From a74f72ffc08eea40be9e42377c7e9b58e1049033 Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Mon, 18 Mar 2024 14:54:59 -0300 Subject: [PATCH 01/41] refactor: move files to agent Signed-off-by: Gustavo Inacio --- tap-agent/src/agent.rs | 9 ++++++--- tap-agent/src/{tap => agent}/sender_account.rs | 8 ++++---- .../src/{tap => agent}/sender_accounts_manager.rs | 2 +- tap-agent/src/{tap => agent}/sender_allocation.rs | 5 ++--- tap-agent/src/tap/mod.rs | 11 ++++------- 5 files changed, 17 insertions(+), 18 deletions(-) rename tap-agent/src/{tap => agent}/sender_account.rs (99%) rename tap-agent/src/{tap => agent}/sender_accounts_manager.rs (99%) rename tap-agent/src/{tap => agent}/sender_allocation.rs (99%) diff --git a/tap-agent/src/agent.rs b/tap-agent/src/agent.rs index 345e56961..a64e3eb27 100644 --- a/tap-agent/src/agent.rs +++ b/tap-agent/src/agent.rs @@ -8,9 +8,12 @@ use indexer_common::prelude::{ escrow_accounts, indexer_allocations, DeploymentDetails, SubgraphClient, }; -use crate::{ - aggregator_endpoints, config, database, tap::sender_accounts_manager::SenderAccountsManager, -}; +use crate::{aggregator_endpoints, config, database}; +use sender_accounts_manager::SenderAccountsManager; + +mod sender_account; +pub mod sender_accounts_manager; +mod sender_allocation; pub async fn start_agent(config: &'static config::Cli) -> SenderAccountsManager { let pgpool = database::connect(&config.postgres).await; diff --git a/tap-agent/src/tap/sender_account.rs b/tap-agent/src/agent/sender_account.rs similarity index 99% rename from tap-agent/src/tap/sender_account.rs rename to tap-agent/src/agent/sender_account.rs index 653149f59..1a4750aff 100644 --- a/tap-agent/src/tap/sender_account.rs +++ b/tap-agent/src/agent/sender_account.rs @@ -21,10 +21,10 @@ use tokio::time::sleep; use tokio::{select, sync::Notify, time}; use tracing::{error, warn}; -use crate::config::{self}; -use crate::tap::{ - escrow_adapter::EscrowAdapter, sender_accounts_manager::NewReceiptNotification, - sender_allocation::SenderAllocation, unaggregated_receipts::UnaggregatedReceipts, +use super::{sender_accounts_manager::NewReceiptNotification, sender_allocation::SenderAllocation}; +use crate::{ + config::{self}, + tap::{escrow_adapter::EscrowAdapter, unaggregated_receipts::UnaggregatedReceipts}, }; #[derive(Clone, EnumAsInner)] diff --git a/tap-agent/src/tap/sender_accounts_manager.rs b/tap-agent/src/agent/sender_accounts_manager.rs similarity index 99% rename from tap-agent/src/tap/sender_accounts_manager.rs rename to tap-agent/src/agent/sender_accounts_manager.rs index 9800f898e..956ec125e 100644 --- a/tap-agent/src/tap/sender_accounts_manager.rs +++ b/tap-agent/src/agent/sender_accounts_manager.rs @@ -16,8 +16,8 @@ use sqlx::{postgres::PgListener, PgPool}; use thegraph::types::Address; use tracing::{error, warn}; +use super::sender_account::SenderAccount; use crate::config; -use crate::tap::sender_account::SenderAccount; #[derive(Deserialize, Debug)] pub struct NewReceiptNotification { diff --git a/tap-agent/src/tap/sender_allocation.rs b/tap-agent/src/agent/sender_allocation.rs similarity index 99% rename from tap-agent/src/tap/sender_allocation.rs rename to tap-agent/src/agent/sender_allocation.rs index e292ec70a..76d9c57fa 100644 --- a/tap-agent/src/tap/sender_allocation.rs +++ b/tap-agent/src/agent/sender_allocation.rs @@ -29,12 +29,11 @@ use tracing::{error, warn}; use crate::{ config::{self}, + tap::context::{checks::Signature, TapAgentContext}, + tap::{context::checks::AllocationId, escrow_adapter::EscrowAdapter}, tap::{signers_trimmed, unaggregated_receipts::UnaggregatedReceipts}, }; -use super::context::{checks::Signature, TapAgentContext}; -use super::{context::checks::AllocationId, escrow_adapter::EscrowAdapter}; - type TapManager = tap_core::manager::Manager; /// Manages unaggregated fees and the TAP lifecyle for a specific (allocation, sender) pair. diff --git a/tap-agent/src/tap/mod.rs b/tap-agent/src/tap/mod.rs index bb72d83cc..824493a90 100644 --- a/tap-agent/src/tap/mod.rs +++ b/tap-agent/src/tap/mod.rs @@ -7,17 +7,14 @@ use eventuals::Eventual; use indexer_common::escrow_accounts::EscrowAccounts; use thegraph::types::Address; -mod context; -mod escrow_adapter; -mod sender_account; -pub mod sender_accounts_manager; -mod sender_allocation; -mod unaggregated_receipts; +pub mod context; +pub mod escrow_adapter; +pub mod unaggregated_receipts; #[cfg(test)] pub mod test_utils; -async fn signers_trimmed( +pub async fn signers_trimmed( escrow_accounts: &Eventual, sender: Address, ) -> Result, anyhow::Error> { From 8a1c6eaf4e008cb6c718086402384a382d17d405 Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Mon, 18 Mar 2024 14:55:33 -0300 Subject: [PATCH 02/41] chore: add ractor Signed-off-by: Gustavo Inacio --- Cargo.lock | 17 +++++++++++++++++ tap-agent/Cargo.toml | 1 + 2 files changed, 18 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index a5c20bed6..6584cecf5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3166,6 +3166,7 @@ dependencies = [ "indexer-common", "jsonrpsee 0.20.2", "lazy_static", + "ractor", "reqwest", "serde", "serde_json", @@ -4843,6 +4844,21 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "ractor" +version = "0.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd876f0d609ba2ddc8a36136e9b81299312bd9fc9b71131381d16c9ce8e495a" +dependencies = [ + "async-trait", + "dashmap", + "futures", + "once_cell", + "rand 0.8.5", + "tokio", + "tracing", +] + [[package]] name = "radium" version = "0.7.0" @@ -6579,6 +6595,7 @@ dependencies = [ "signal-hook-registry", "socket2 0.5.4", "tokio-macros", + "tracing", "windows-sys", ] diff --git a/tap-agent/Cargo.toml b/tap-agent/Cargo.toml index 1200442ea..3b5db4cd6 100644 --- a/tap-agent/Cargo.toml +++ b/tap-agent/Cargo.toml @@ -50,6 +50,7 @@ tracing-subscriber = { version = "0.3", features = [ enum-as-inner = "0.6.0" ethers = "2.0.13" typetag = "0.2.14" +ractor = "0.9.7" [dev-dependencies] ethers-signers = "2.0.8" From 59c93021262fabc3d11e5b22d1a614e7e832b67a Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Mon, 18 Mar 2024 14:57:01 -0300 Subject: [PATCH 03/41] refactor: move unaggregated_receipts to agent Signed-off-by: Gustavo Inacio --- tap-agent/src/agent.rs | 1 + tap-agent/src/agent/sender_account.rs | 3 ++- tap-agent/src/agent/sender_allocation.rs | 3 ++- tap-agent/src/{tap => agent}/unaggregated_receipts.rs | 0 tap-agent/src/tap/mod.rs | 1 - 5 files changed, 5 insertions(+), 3 deletions(-) rename tap-agent/src/{tap => agent}/unaggregated_receipts.rs (100%) diff --git a/tap-agent/src/agent.rs b/tap-agent/src/agent.rs index a64e3eb27..4ca3bba5f 100644 --- a/tap-agent/src/agent.rs +++ b/tap-agent/src/agent.rs @@ -14,6 +14,7 @@ use sender_accounts_manager::SenderAccountsManager; mod sender_account; pub mod sender_accounts_manager; mod sender_allocation; +mod unaggregated_receipts; pub async fn start_agent(config: &'static config::Cli) -> SenderAccountsManager { let pgpool = database::connect(&config.postgres).await; diff --git a/tap-agent/src/agent/sender_account.rs b/tap-agent/src/agent/sender_account.rs index 1a4750aff..e09c655d2 100644 --- a/tap-agent/src/agent/sender_account.rs +++ b/tap-agent/src/agent/sender_account.rs @@ -22,9 +22,10 @@ use tokio::{select, sync::Notify, time}; use tracing::{error, warn}; use super::{sender_accounts_manager::NewReceiptNotification, sender_allocation::SenderAllocation}; +use crate::agent::unaggregated_receipts::UnaggregatedReceipts; use crate::{ config::{self}, - tap::{escrow_adapter::EscrowAdapter, unaggregated_receipts::UnaggregatedReceipts}, + tap::escrow_adapter::EscrowAdapter, }; #[derive(Clone, EnumAsInner)] diff --git a/tap-agent/src/agent/sender_allocation.rs b/tap-agent/src/agent/sender_allocation.rs index 76d9c57fa..62f9e1cd3 100644 --- a/tap-agent/src/agent/sender_allocation.rs +++ b/tap-agent/src/agent/sender_allocation.rs @@ -27,11 +27,12 @@ use thegraph::types::Address; use tokio::sync::Mutex as TokioMutex; use tracing::{error, warn}; +use crate::agent::unaggregated_receipts::UnaggregatedReceipts; use crate::{ config::{self}, tap::context::{checks::Signature, TapAgentContext}, + tap::signers_trimmed, tap::{context::checks::AllocationId, escrow_adapter::EscrowAdapter}, - tap::{signers_trimmed, unaggregated_receipts::UnaggregatedReceipts}, }; type TapManager = tap_core::manager::Manager; diff --git a/tap-agent/src/tap/unaggregated_receipts.rs b/tap-agent/src/agent/unaggregated_receipts.rs similarity index 100% rename from tap-agent/src/tap/unaggregated_receipts.rs rename to tap-agent/src/agent/unaggregated_receipts.rs diff --git a/tap-agent/src/tap/mod.rs b/tap-agent/src/tap/mod.rs index 824493a90..3802cf8b7 100644 --- a/tap-agent/src/tap/mod.rs +++ b/tap-agent/src/tap/mod.rs @@ -9,7 +9,6 @@ use thegraph::types::Address; pub mod context; pub mod escrow_adapter; -pub mod unaggregated_receipts; #[cfg(test)] pub mod test_utils; From 4683b1b57d6116adc1d2ff07bef69aa9ee749486 Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Mon, 18 Mar 2024 20:48:25 -0300 Subject: [PATCH 04/41] refactor: add allocation id tracker --- tap-agent/src/agent.rs | 1 + tap-agent/src/agent/allocation_id_tracker.rs | 46 ++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 tap-agent/src/agent/allocation_id_tracker.rs diff --git a/tap-agent/src/agent.rs b/tap-agent/src/agent.rs index 4ca3bba5f..61716d4e7 100644 --- a/tap-agent/src/agent.rs +++ b/tap-agent/src/agent.rs @@ -15,6 +15,7 @@ mod sender_account; pub mod sender_accounts_manager; mod sender_allocation; mod unaggregated_receipts; +mod allocation_id_tracker; pub async fn start_agent(config: &'static config::Cli) -> SenderAccountsManager { let pgpool = database::connect(&config.postgres).await; diff --git a/tap-agent/src/agent/allocation_id_tracker.rs b/tap-agent/src/agent/allocation_id_tracker.rs new file mode 100644 index 000000000..802606a12 --- /dev/null +++ b/tap-agent/src/agent/allocation_id_tracker.rs @@ -0,0 +1,46 @@ +use alloy_primitives::Address; +use std::collections::{BTreeMap, HashMap}; + +#[derive(Debug)] +pub struct AllocationIdTracker { + id_to_fee: HashMap, + fee_to_count: BTreeMap, + total_fee: u128, +} + +impl AllocationIdTracker { + pub fn new() -> Self { + AllocationIdTracker { + id_to_fee: HashMap::new(), + fee_to_count: BTreeMap::new(), + total_fee: 0, + } + } + + pub fn add_or_update(&mut self, id: Address, fee: u128) { + if let Some(&old_fee) = self.id_to_fee.get(&id) { + self.total_fee -= old_fee; + *self.fee_to_count.get_mut(&old_fee).unwrap() -= 1; + if self.fee_to_count[&old_fee] == 0 { + self.fee_to_count.remove(&old_fee); + } + } + + self.id_to_fee.insert(id, fee); + self.total_fee += fee; + *self.fee_to_count.entry(fee).or_insert(0) += 1; + } + + pub fn get_heaviest_allocation_id(&self) -> Option
{ + self.fee_to_count.iter().next_back().and_then(|(&fee, _)| { + self.id_to_fee + .iter() + .find(|(_, &f)| f == fee) + .map(|(&id, _)| id) + }) + } + + pub fn get_total_fee(&self) -> u128 { + self.total_fee + } +} From 80332adeb079c08fda26dad1af8509db4542e5d2 Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Mon, 18 Mar 2024 20:51:20 -0300 Subject: [PATCH 05/41] refactor(wip): use actors for manager, account and allocation --- tap-agent/src/agent/sender_account.rs | 467 ++++-------------- .../src/agent/sender_accounts_manager.rs | 327 ++++++------ tap-agent/src/agent/sender_allocation.rs | 143 ++++-- 3 files changed, 341 insertions(+), 596 deletions(-) diff --git a/tap-agent/src/agent/sender_account.rs b/tap-agent/src/agent/sender_account.rs index e09c655d2..571a98d13 100644 --- a/tap-agent/src/agent/sender_account.rs +++ b/tap-agent/src/agent/sender_account.rs @@ -1,270 +1,112 @@ // Copyright 2023-, GraphOps and Semiotic Labs. // SPDX-License-Identifier: Apache-2.0 -use std::sync::Mutex as StdMutex; -use std::{ - cmp::max, - collections::{HashMap, HashSet}, - sync::Arc, - time::Duration, -}; +use std::collections::HashSet; use alloy_sol_types::Eip712Domain; -use anyhow::{anyhow, Result}; -use enum_as_inner::EnumAsInner; +use anyhow::Result; use eventuals::Eventual; use indexer_common::{escrow_accounts::EscrowAccounts, prelude::SubgraphClient}; +use ractor::{call, Actor, ActorProcessingErr, ActorRef}; use sqlx::PgPool; use thegraph::types::Address; -use tokio::sync::Mutex as TokioMutex; -use tokio::time::sleep; -use tokio::{select, sync::Notify, time}; -use tracing::{error, warn}; -use super::{sender_accounts_manager::NewReceiptNotification, sender_allocation::SenderAllocation}; +use super::sender_allocation::SenderAllocation; +use crate::agent::allocation_id_tracker::AllocationIdTracker; +use crate::agent::sender_allocation::SenderAllocationMsg; use crate::agent::unaggregated_receipts::UnaggregatedReceipts; use crate::{ config::{self}, tap::escrow_adapter::EscrowAdapter, }; -#[derive(Clone, EnumAsInner)] -enum AllocationState { - Active(Arc), - Ineligible(Arc), +pub enum SenderAccountMessage { + CreateSenderAllocation(Address), + RemoveSenderAccount, + UpdateReceiptFees(Address, UnaggregatedReceipts), } -/// The inner state of a SenderAccount. This is used to store an Arc state for spawning async tasks. -pub struct Inner { +/// A SenderAccount manages the receipts accounting between the indexer and the sender across +/// multiple allocations. +/// +/// Manages the lifecycle of Scalar TAP for the SenderAccount, including: +/// - Monitoring new receipts and keeping track of the cumulative unaggregated fees across +/// allocations. +/// - Requesting RAVs from the sender's TAP aggregator once the cumulative unaggregated fees reach a +/// certain threshold. +/// - Requesting the last RAV from the sender's TAP aggregator for all EOL allocations. +pub struct SenderAccount { + escrow_accounts: Eventual, + escrow_subgraph: &'static SubgraphClient, + escrow_adapter: EscrowAdapter, + tap_eip712_domain_separator: Eip712Domain, config: &'static config::Cli, pgpool: PgPool, - allocations: Arc>>, sender: Address, sender_aggregator_endpoint: String, - unaggregated_fees: Arc>, - unaggregated_receipts_guard: Arc>, -} - -/// Wrapper around a `Notify` to trigger RAV requests. -/// This gives a better understanding of what is happening -/// because the methods names are more explicit. -#[derive(Clone)] -pub struct RavTrigger(Arc); - -impl RavTrigger { - pub fn new() -> Self { - Self(Arc::new(Notify::new())) - } - - /// Trigger a RAV request if there are any waiters. - /// In case there are no waiters, nothing happens. - pub fn trigger_rav(&self) { - self.0.notify_waiters(); - } - - /// Wait for a RAV trigger. - pub async fn wait_for_rav_request(&self) { - self.0.notified().await; - } - - /// Trigger a RAV request. In case there are no waiters, the request is queued - /// and is executed on the next call to wait_for_rav_trigger(). - pub fn trigger_next_rav(&self) { - self.0.notify_one(); - } } -impl Inner { - async fn rav_requester(&self, trigger: RavTrigger) { - loop { - trigger.wait_for_rav_request().await; - - if let Err(error) = self.rav_requester_single().await { - // If an error occoured, we shouldn't retry right away, so we wait for a bit. - error!( - "Error while requesting RAV for sender {}: {}", - self.sender, error - ); - // simpler for now, maybe we can add a backoff strategy later - sleep(Duration::from_secs(5)).await; - continue; - } +#[async_trait::async_trait] +impl Actor for SenderAccount { + type Msg = SenderAccountMessage; + type State = AllocationIdTracker; + type Arguments = HashSet
; - // Check if we already need to send another RAV request. - let unaggregated_fees = self.unaggregated_fees.lock().unwrap().clone(); - if unaggregated_fees.value >= self.config.tap.rav_request_trigger_value.into() { - // If so, "self-notify" to trigger another RAV request. - trigger.trigger_next_rav(); + async fn pre_start( + &self, + _myself: ActorRef, + _args: Self::Arguments, + ) -> std::result::Result { + // todo look for allocations and create sender_allocations - warn!( - "Sender {} has {} unaggregated fees immediately after a RAV request, which is - over the trigger value. Triggering another RAV request.", - self.sender, unaggregated_fees.value, - ); - } - } + // todo use the arguments to spawn the initial sender allocations + Ok(AllocationIdTracker::new()) } - async fn rav_requester_finalize(&self, finalize_trigger: RavTrigger) { - loop { - // Wait for either 5 minutes or a notification that we need to try to finalize - // allocation receipts. - select! { - _ = time::sleep(Duration::from_secs(300)) => (), - _ = finalize_trigger.wait_for_rav_request() => () - } - - // Get a quick snapshot of the current finalizing allocations. They are - // Arcs, so it should be cheap. - let allocations_finalizing = self - .allocations - .lock() - .unwrap() - .values() - .filter(|a| matches!(a, AllocationState::Ineligible(_))) - .map(|a| a.as_ineligible().unwrap()) - .cloned() - .collect::>(); - - for allocation in allocations_finalizing { - if let Err(e) = allocation.rav_requester_single().await { - error!( - "Error while requesting RAV for sender {} and allocation {}: {}", - self.sender, - allocation.get_allocation_id(), - e - ); - continue; - } - - if let Err(e) = allocation.mark_rav_last().await { - error!( - "Error while marking allocation {} as last for sender {}: {}", - allocation.get_allocation_id(), - self.sender, - e - ); - continue; + async fn handle( + &self, + myself: ActorRef, + message: Self::Msg, + state: &mut Self::State, + ) -> std::result::Result<(), ActorProcessingErr> { + match message { + SenderAccountMessage::RemoveSenderAccount => myself.stop(None), + SenderAccountMessage::UpdateReceiptFees(allocation_id, unaggregated_fees) => { + state.add_or_update(allocation_id, unaggregated_fees.value); + + if state.get_total_fee() >= self.config.tap.rav_request_trigger_value.into() { + self.rav_requester_single(state).await?; } - - // Remove the allocation from the finalizing map. - self.allocations - .lock() - .unwrap() - .remove(&allocation.get_allocation_id()); } - } - } - - /// Does a single RAV request for the sender's allocation with the highest unaggregated fees - async fn rav_requester_single(&self) -> Result<()> { - let heaviest_allocation = self.get_heaviest_allocation().ok_or(anyhow! { - "Error while getting allocation with most unaggregated fees", - })?; - heaviest_allocation - .rav_requester_single() - .await - .map_err(|e| { - anyhow! { - "Error while requesting RAV for sender {} and allocation {}: {}", + SenderAccountMessage::CreateSenderAllocation(allocation_id) => { + let sender_allocation = SenderAllocation::new( + self.config, + self.pgpool.clone(), + allocation_id, self.sender, - heaviest_allocation.get_allocation_id(), - e - } - })?; - - self.recompute_unaggregated_fees().await; - - Ok(()) - } + self.escrow_accounts.clone(), + self.escrow_subgraph, + self.escrow_adapter.clone(), + self.tap_eip712_domain_separator.clone(), + self.sender_aggregator_endpoint.clone(), + myself.clone(), + ) + .await; + let sender_id = self.sender.clone(); - /// Returns the allocation with the highest unaggregated fees value. - /// If there are no active allocations, returns None. - fn get_heaviest_allocation(&self) -> Option> { - // Get a quick snapshot of all allocations. They are Arcs, so it should be cheap, - // and we don't want to hold the lock for too long. - let allocations: Vec<_> = self.allocations.lock().unwrap().values().cloned().collect(); - - let mut heaviest_allocation = (None, 0u128); - for allocation in allocations { - let allocation: Arc = match allocation { - AllocationState::Active(a) => a, - AllocationState::Ineligible(a) => a, - }; - let fees = allocation.get_unaggregated_fees().value; - if fees > heaviest_allocation.1 { - heaviest_allocation = (Some(allocation), fees); + let (_actor, _handle) = SenderAllocation::spawn_linked( + Some(format!("{sender_id}:{allocation_id}")), + sender_allocation, + (), + myself.get_cell(), + ) + .await?; } } - - heaviest_allocation.0 - } - - /// Recompute the sender's total unaggregated fees value and last receipt ID. - async fn recompute_unaggregated_fees(&self) { - // Make sure to pause the handling of receipt notifications while we update the unaggregated - // fees. - let _guard = self.unaggregated_receipts_guard.lock().await; - - // Similar pattern to get_heaviest_allocation(). - let allocations: Vec<_> = self.allocations.lock().unwrap().values().cloned().collect(); - - // Gather the unaggregated fees from all allocations and sum them up. - let mut unaggregated_fees = self.unaggregated_fees.lock().unwrap(); - *unaggregated_fees = UnaggregatedReceipts::default(); // Reset to 0. - for allocation in allocations { - let allocation: Arc = match allocation { - AllocationState::Active(a) => a, - AllocationState::Ineligible(a) => a, - }; - - let uf = allocation.get_unaggregated_fees(); - *unaggregated_fees = UnaggregatedReceipts { - value: self.fees_add(unaggregated_fees.value, uf.value), - last_id: max(unaggregated_fees.last_id, uf.last_id), - }; - } - } - - /// Safe add the fees to the unaggregated fees value, log an error if there is an overflow and - /// set the unaggregated fees value to u128::MAX. - fn fees_add(&self, total_unaggregated_fees: u128, value_increment: u128) -> u128 { - total_unaggregated_fees - .checked_add(value_increment) - .unwrap_or_else(|| { - // This should never happen, but if it does, we want to know about it. - error!( - "Overflow when adding receipt value {} to total unaggregated fees {} for \ - sender {}. Setting total unaggregated fees to u128::MAX.", - value_increment, total_unaggregated_fees, self.sender - ); - u128::MAX - }) + Ok(()) } } -/// A SenderAccount manages the receipts accounting between the indexer and the sender across -/// multiple allocations. -/// -/// Manages the lifecycle of Scalar TAP for the SenderAccount, including: -/// - Monitoring new receipts and keeping track of the cumulative unaggregated fees across -/// allocations. -/// - Requesting RAVs from the sender's TAP aggregator once the cumulative unaggregated fees reach a -/// certain threshold. -/// - Requesting the last RAV from the sender's TAP aggregator for all EOL allocations. -pub struct SenderAccount { - inner: Arc, - escrow_accounts: Eventual, - escrow_subgraph: &'static SubgraphClient, - escrow_adapter: EscrowAdapter, - tap_eip712_domain_separator: Eip712Domain, - rav_requester_task: tokio::task::JoinHandle<()>, - rav_trigger: RavTrigger, - rav_requester_finalize_task: tokio::task::JoinHandle<()>, - rav_finalize_trigger: RavTrigger, - unaggregated_receipts_guard: Arc>, -} - impl SenderAccount { #[allow(clippy::too_many_arguments)] pub fn new( @@ -276,164 +118,39 @@ impl SenderAccount { tap_eip712_domain_separator: Eip712Domain, sender_aggregator_endpoint: String, ) -> Self { - let unaggregated_receipts_guard = Arc::new(TokioMutex::new(())); - let escrow_adapter = EscrowAdapter::new(escrow_accounts.clone(), sender_id); - let inner = Arc::new(Inner { - config, - pgpool, - allocations: Arc::new(StdMutex::new(HashMap::new())), - sender: sender_id, - sender_aggregator_endpoint, - unaggregated_fees: Arc::new(StdMutex::new(UnaggregatedReceipts::default())), - unaggregated_receipts_guard: unaggregated_receipts_guard.clone(), - }); - - let rav_trigger = RavTrigger::new(); - let rav_requester_task = tokio::spawn({ - let inner = inner.clone(); - let rav_trigger = rav_trigger.clone(); - async move { - inner.rav_requester(rav_trigger).await; - } - }); - - let rav_finalize_trigger = RavTrigger::new(); - let rav_requester_finalize_task = tokio::spawn({ - let inner = inner.clone(); - let rav_finalize_trigger = rav_finalize_trigger.clone(); - async move { - inner.rav_requester_finalize(rav_finalize_trigger).await; - } - }); - Self { - inner: inner.clone(), escrow_accounts, escrow_subgraph, escrow_adapter, tap_eip712_domain_separator, - rav_requester_task, - rav_trigger, - rav_requester_finalize_task, - rav_finalize_trigger, - unaggregated_receipts_guard, - } - } - - /// Update the sender's allocations to match the target allocations. - pub async fn update_allocations(&self, target_allocations: HashSet
) { - { - let mut allocations = self.inner.allocations.lock().unwrap(); - let mut allocations_to_finalize = false; - - // Make allocations that are no longer to be active `AllocationState::Ineligible`. - for (allocation_id, allocation_state) in allocations.iter_mut() { - if !target_allocations.contains(allocation_id) { - match allocation_state { - AllocationState::Active(allocation) => { - *allocation_state = AllocationState::Ineligible(allocation.clone()); - allocations_to_finalize = true; - } - AllocationState::Ineligible(_) => { - // Allocation is already ineligible, do nothing. - } - } - } - } - - if allocations_to_finalize { - self.rav_finalize_trigger.trigger_rav(); - } - } - - // Add new allocations. - for allocation_id in target_allocations { - let sender_allocation = AllocationState::Active(Arc::new( - SenderAllocation::new( - self.inner.config, - self.inner.pgpool.clone(), - allocation_id, - self.inner.sender, - self.escrow_accounts.clone(), - self.escrow_subgraph, - self.escrow_adapter.clone(), - self.tap_eip712_domain_separator.clone(), - self.inner.sender_aggregator_endpoint.clone(), - ) - .await, - )); - if let std::collections::hash_map::Entry::Vacant(e) = - self.inner.allocations.lock().unwrap().entry(allocation_id) - { - e.insert(sender_allocation); - } + sender_aggregator_endpoint, + config, + pgpool, + sender: sender_id, } } - pub async fn handle_new_receipt_notification( + async fn rav_requester_single( &self, - new_receipt_notification: NewReceiptNotification, - ) { - // Make sure to pause the handling of receipt notifications while we update the unaggregated - // fees. - let _guard = self.unaggregated_receipts_guard.lock().await; - - let allocation_state = self - .inner - .allocations - .lock() - .unwrap() - .get(&new_receipt_notification.allocation_id) - .cloned(); - - if let Some(AllocationState::Active(allocation)) = allocation_state { - // Try to add the receipt value to the allocation's unaggregated fees value. - // If the fees were not added, it means the receipt was already processed, so we - // don't need to do anything. - if allocation - .fees_add(new_receipt_notification.value, new_receipt_notification.id) - .await - { - // Add the receipt value to the allocation's unaggregated fees value. - allocation - .fees_add(new_receipt_notification.value, new_receipt_notification.id) - .await; - // Add the receipt value to the sender's unaggregated fees value. - let mut unaggregated_fees = self.inner.unaggregated_fees.lock().unwrap(); - *unaggregated_fees = UnaggregatedReceipts { - value: self - .inner - .fees_add(unaggregated_fees.value, new_receipt_notification.value), - last_id: new_receipt_notification.id, - }; - - // Check if we need to trigger a RAV request. - if unaggregated_fees.value >= self.inner.config.tap.rav_request_trigger_value.into() - { - self.rav_trigger.trigger_rav(); - } - } - } else { - error!( - "Received a new receipt notification for allocation {} that doesn't exist \ - or is ineligible for sender {}.", - new_receipt_notification.allocation_id, self.inner.sender - ); - } - } + heaviest_allocation: &mut AllocationIdTracker, + ) -> Result<()> { + let Some(allocation_id) = heaviest_allocation.get_heaviest_allocation_id() else { + anyhow::bail!("Error while getting allocation with most unaggregated fees"); + }; + let sender_id = self.sender; + let allocation = + ActorRef::::where_is(format!("{sender_id}:{allocation_id}")); - pub async fn recompute_unaggregated_fees(&self) { - self.inner.recompute_unaggregated_fees().await - } -} + let Some(allocation) = allocation else { + anyhow::bail!("Error while getting allocation with most unaggregated fees"); + }; + // we call and wait for the response so we don't process anymore update + let result = call!(allocation, SenderAllocationMsg::TriggerRAVRequest)?; -// Abort tasks on Drop -impl Drop for SenderAccount { - fn drop(&mut self) { - self.rav_requester_task.abort(); - self.rav_requester_finalize_task.abort(); + heaviest_allocation.add_or_update(allocation_id, result.value); + Ok(()) } } diff --git a/tap-agent/src/agent/sender_accounts_manager.rs b/tap-agent/src/agent/sender_accounts_manager.rs index 956ec125e..da5fb0714 100644 --- a/tap-agent/src/agent/sender_accounts_manager.rs +++ b/tap-agent/src/agent/sender_accounts_manager.rs @@ -2,21 +2,22 @@ // SPDX-License-Identifier: Apache-2.0 use std::collections::HashSet; -use std::sync::Mutex as StdMutex; -use std::{collections::HashMap, str::FromStr, sync::Arc}; +use std::{collections::HashMap, str::FromStr}; +use crate::agent::sender_allocation::SenderAllocationMsg; use alloy_sol_types::Eip712Domain; use anyhow::anyhow; use anyhow::Result; use eventuals::{Eventual, EventualExt, PipeHandle}; use indexer_common::escrow_accounts::EscrowAccounts; use indexer_common::prelude::{Allocation, SubgraphClient}; +use ractor::{Actor, ActorProcessingErr, ActorRef}; use serde::Deserialize; use sqlx::{postgres::PgListener, PgPool}; use thegraph::types::Address; use tracing::{error, warn}; -use super::sender_account::SenderAccount; +use super::sender_account::{SenderAccount, SenderAccountMessage}; use crate::config; #[derive(Deserialize, Debug)] @@ -29,18 +30,8 @@ pub struct NewReceiptNotification { } pub struct SenderAccountsManager { - _inner: Arc, - new_receipts_watcher_handle: tokio::task::JoinHandle<()>, - _eligible_allocations_senders_pipe: PipeHandle, -} - -/// Inner struct for SenderAccountsManager. This is used to store an Arc state for spawning async -/// tasks. -struct Inner { config: &'static config::Cli, pgpool: PgPool, - /// Map of sender_address to SenderAllocation. - sender_accounts: Arc>>>, indexer_allocations: Eventual>, escrow_accounts: Eventual, escrow_subgraph: &'static SubgraphClient, @@ -48,58 +39,120 @@ struct Inner { sender_aggregator_endpoints: HashMap, } -impl Inner { - async fn update_sender_accounts( +pub enum NetworkMessage { + UpdateSenderAccounts(HashSet
), + CreateSenderAccount(Address, HashSet
), +} + +pub struct State { + sender_ids: HashSet
, + new_receipts_watcher_handle: tokio::task::JoinHandle<()>, + _eligible_allocations_senders_pipe: PipeHandle, +} + +#[async_trait::async_trait] +impl Actor for SenderAccountsManager { + type Msg = NetworkMessage; + type State = State; + type Arguments = (); + + async fn pre_start( &self, - indexer_allocations: HashMap, - target_senders: HashSet
, - ) -> Result<()> { - let eligible_allocations: HashSet
= indexer_allocations.keys().copied().collect(); - let mut sender_accounts_copy = self.sender_accounts.lock().unwrap().clone(); - - // For all Senders that are not in the target_senders HashSet, set all their allocations to - // ineligible. That will trigger a finalization of all their receipts. - for (sender_id, sender_account) in sender_accounts_copy.iter() { - if !target_senders.contains(sender_id) { - sender_account.update_allocations(HashSet::new()).await; - } + myself: ActorRef, + _: Self::Arguments, + ) -> std::result::Result { + let mut pglistener = PgListener::connect_with(&self.pgpool.clone()) + .await + .unwrap(); + pglistener + .listen("scalar_tap_receipt_notification") + .await + .expect( + "should be able to subscribe to Postgres Notify events on the channel \ + 'scalar_tap_receipt_notification'", + ); + // Start the new_receipts_watcher task that will consume from the `pglistener` + let new_receipts_watcher_handle = tokio::spawn(Self::new_receipts_watcher( + pglistener, + self.escrow_accounts.clone(), + )); + let clone = myself.clone(); + let eligible_allocations_senders_pipe = + self.escrow_accounts + .clone() + .pipe_async(move |escrow_accounts| { + let myself = clone.clone(); + + async move { + myself + .cast(NetworkMessage::UpdateSenderAccounts( + escrow_accounts.get_senders(), + )) + .unwrap_or_else(|e| { + error!("Error while updating sender_accounts: {:?}", e); + }); + } + }); + let sender_allocation = self.get_pending_sender_allocation_id().await; + for (sender_id, allocation_ids) in sender_allocation { + myself.cast(NetworkMessage::CreateSenderAccount( + sender_id, + allocation_ids, + ))?; } - // Get or create SenderAccount instances for all currently eligible - // senders. - for sender_id in &target_senders { - let sender = - sender_accounts_copy - .entry(*sender_id) - .or_insert(Arc::new(SenderAccount::new( - self.config, - self.pgpool.clone(), - *sender_id, - self.escrow_accounts.clone(), - self.escrow_subgraph, - self.tap_eip712_domain_separator.clone(), - self.sender_aggregator_endpoints - .get(sender_id) - .ok_or_else(|| { - anyhow!( - "No sender_aggregator_endpoint found for sender {}", - sender_id - ) - })? - .clone(), - ))); - - // Update sender's allocations - sender - .update_allocations(eligible_allocations.clone()) - .await; - } + Ok(State { + sender_ids: HashSet::new(), + new_receipts_watcher_handle, + _eligible_allocations_senders_pipe: eligible_allocations_senders_pipe, + }) + } + async fn post_stop( + &self, + _myself: ActorRef, + state: &mut Self::State, + ) -> std::result::Result<(), ActorProcessingErr> { + // Abort the notification watcher on drop. Otherwise it may panic because the PgPool could + // get dropped before. (Observed in tests) + state.new_receipts_watcher_handle.abort(); + Ok(()) + } - // Replace the sender_accounts with the updated sender_accounts_copy - *self.sender_accounts.lock().unwrap() = sender_accounts_copy; + async fn handle( + &self, + myself: ActorRef, + msg: Self::Msg, + state: &mut Self::State, + ) -> std::result::Result<(), ActorProcessingErr> { + match msg { + NetworkMessage::UpdateSenderAccounts(target_senders) => { + // Create new sender accounts + for sender in target_senders.difference(&state.sender_ids) { + myself.cast(NetworkMessage::CreateSenderAccount(*sender, HashSet::new()))?; + } - // TODO: remove Sender instances that are finished. Ideally done in another async task? + // Remove sender accounts + for sender in state.sender_ids.difference(&target_senders) { + if let Some(sender_handle) = + ActorRef::::where_is(sender.to_string()) + { + sender_handle.cast(SenderAccountMessage::RemoveSenderAccount)?; + } + } + state.sender_ids = target_senders; + } + NetworkMessage::CreateSenderAccount(sender_id, allocation_ids) => { + let sender_account = self.new_sender_account(&sender_id)?; + SenderAccount::spawn_linked( + Some(sender_id.to_string()), + sender_account, + allocation_ids, + myself.get_cell(), + ) + .await?; + } + } Ok(()) } } @@ -114,32 +167,19 @@ impl SenderAccountsManager { tap_eip712_domain_separator: Eip712Domain, sender_aggregator_endpoints: HashMap, ) -> Self { - let inner = Arc::new(Inner { + Self { config, pgpool, - sender_accounts: Arc::new(StdMutex::new(HashMap::new())), indexer_allocations, escrow_accounts, escrow_subgraph, tap_eip712_domain_separator, sender_aggregator_endpoints, - }); - - // Listen to pg_notify events. We start it before updating the unaggregated_fees for all - // SenderAccount instances, so that we don't miss any receipts. PG will buffer the\ - // notifications until we start consuming them with `new_receipts_watcher`. - let mut pglistener = PgListener::connect_with(&inner.pgpool.clone()) - .await - .unwrap(); - pglistener - .listen("scalar_tap_receipt_notification") - .await - .expect( - "should be able to subscribe to Postgres Notify events on the channel \ - 'scalar_tap_receipt_notification'", - ); + } + } - let escrow_accounts_snapshot = inner + async fn get_pending_sender_allocation_id(&self) -> HashMap> { + let escrow_accounts_snapshot = self .escrow_accounts .value() .await @@ -169,7 +209,7 @@ impl SenderAccountsManager { FROM scalar_tap_receipts AS top "# ) - .fetch_all(&inner.pgpool) + .fetch_all(&self.pgpool) .await .expect("should be able to fetch pending receipts from the database"); @@ -211,7 +251,7 @@ impl SenderAccountsManager { FROM scalar_tap_ravs AS top "# ) - .fetch_all(&inner.pgpool) + .fetch_all(&self.pgpool) .await .expect("should be able to fetch unfinalized RAVs from the database"); @@ -234,88 +274,32 @@ impl SenderAccountsManager { .or_default() .extend(allocation_ids); } - - // Create SenderAccount instances for all senders that have unfinalized allocations and add - // the allocations to the SenderAccount instances. - let mut sender_accounts = HashMap::new(); - for (sender_id, allocation_ids) in unfinalized_sender_allocations_map { - let sender = sender_accounts - .entry(sender_id) - .or_insert(Arc::new(SenderAccount::new( - config, - inner.pgpool.clone(), - sender_id, - inner.escrow_accounts.clone(), - inner.escrow_subgraph, - inner.tap_eip712_domain_separator.clone(), - inner - .sender_aggregator_endpoints - .get(&sender_id) - .expect("should be able to get sender_aggregator_endpoint for sender") - .clone(), - ))); - - sender.update_allocations(allocation_ids).await; - - sender.recompute_unaggregated_fees().await; - } - // replace the sender_accounts with the updated sender_accounts - *inner.sender_accounts.lock().unwrap() = sender_accounts; - - // Update senders and allocations based on the current state of the network. - // It is important to do this after creating the Sender and SenderAllocation instances based - // on the receipts in the database, because now all ineligible allocation and/or sender that - // we created above will be set for receipt finalization. - inner - .update_sender_accounts( - inner - .indexer_allocations - .value() - .await - .expect("Should get indexer allocations from Eventual"), - escrow_accounts_snapshot.get_senders(), - ) - .await - .expect("Should be able to update_sender_accounts"); - - // Start the new_receipts_watcher task that will consume from the `pglistener` - let new_receipts_watcher_handle = tokio::spawn(Self::new_receipts_watcher( - pglistener, - inner.sender_accounts.clone(), - inner.escrow_accounts.clone(), - )); - - // Start the eligible_allocations_senders_pipe that watches for changes in eligible senders - // and allocations and updates the SenderAccount instances accordingly. - let inner_clone = inner.clone(); - let eligible_allocations_senders_pipe = eventuals::join(( - inner.indexer_allocations.clone(), - inner.escrow_accounts.clone(), + unfinalized_sender_allocations_map + } + fn new_sender_account(&self, sender_id: &Address) -> Result { + Ok(SenderAccount::new( + self.config, + self.pgpool.clone(), + *sender_id, + self.escrow_accounts.clone(), + self.escrow_subgraph, + self.tap_eip712_domain_separator.clone(), + self.sender_aggregator_endpoints + .get(sender_id) + .ok_or_else(|| { + anyhow!( + "No sender_aggregator_endpoint found for sender {}", + sender_id + ) + })? + .clone(), )) - .pipe_async(move |(indexer_allocations, escrow_accounts)| { - let inner = inner_clone.clone(); - async move { - inner - .update_sender_accounts(indexer_allocations, escrow_accounts.get_senders()) - .await - .unwrap_or_else(|e| { - error!("Error while updating sender_accounts: {:?}", e); - }); - } - }); - - Self { - _inner: inner, - new_receipts_watcher_handle, - _eligible_allocations_senders_pipe: eligible_allocations_senders_pipe, - } } /// Continuously listens for new receipt notifications from Postgres and forwards them to the /// corresponding SenderAccount. async fn new_receipts_watcher( mut pglistener: PgListener, - sender_accounts: Arc>>>, escrow_accounts: Eventual, ) { loop { @@ -348,17 +332,16 @@ impl SenderAccountsManager { continue; } }; - - let sender_account = sender_accounts - .lock() - .unwrap() - .get(&sender_address) - .cloned(); - - if let Some(sender_account) = sender_account { - sender_account - .handle_new_receipt_notification(new_receipt_notification) - .await; + let allocation_id = &new_receipt_notification.allocation_id; + + if let Some(sender_allocation) = ActorRef::::where_is(format!( + "{sender_address}:{allocation_id}" + )) { + if let Err(e) = sender_allocation + .cast(SenderAllocationMsg::NewReceipt(new_receipt_notification)) + { + error!("Error while forwarding new receipt notification to sender_allocation: {:?}", e); + } } else { warn!( "No sender_allocation_manager found for sender_address {} to process new \ @@ -370,14 +353,6 @@ impl SenderAccountsManager { } } -impl Drop for SenderAccountsManager { - fn drop(&mut self) { - // Abort the notification watcher on drop. Otherwise it may panic because the PgPool could - // get dropped before. (Observed in tests) - self.new_receipts_watcher_handle.abort(); - } -} - #[cfg(test)] mod tests { @@ -491,7 +466,7 @@ mod tests { // Check that the SenderAccount was created. assert!(sender_account - ._inner + .inner .sender_accounts .lock() .unwrap() @@ -505,7 +480,7 @@ mod tests { // Check that the Sender's allocation moved from active to ineligible. assert!(sender_account - ._inner + .inner .sender_accounts .lock() .unwrap() @@ -514,7 +489,7 @@ mod tests { ._tests_get_allocations_active() .is_empty()); assert!(sender_account - ._inner + .inner .sender_accounts .lock() .unwrap() diff --git a/tap-agent/src/agent/sender_allocation.rs b/tap-agent/src/agent/sender_allocation.rs index 62f9e1cd3..1de5ae184 100644 --- a/tap-agent/src/agent/sender_allocation.rs +++ b/tap-agent/src/agent/sender_allocation.rs @@ -13,6 +13,7 @@ use bigdecimal::num_bigint::BigInt; use eventuals::Eventual; use indexer_common::{escrow_accounts::EscrowAccounts, prelude::SubgraphClient}; use jsonrpsee::{core::client::ClientT, http_client::HttpClientBuilder, rpc_params}; +use ractor::{Actor, ActorProcessingErr, ActorRef, RpcReplyPort}; use sqlx::{types::BigDecimal, PgPool}; use tap_aggregator::jsonrpsee_helpers::JsonRpcResponse; use tap_core::{ @@ -27,6 +28,8 @@ use thegraph::types::Address; use tokio::sync::Mutex as TokioMutex; use tracing::{error, warn}; +use crate::agent::sender_account::SenderAccountMessage; +use crate::agent::sender_accounts_manager::NewReceiptNotification; use crate::agent::unaggregated_receipts::UnaggregatedReceipts; use crate::{ config::{self}, @@ -50,6 +53,98 @@ pub struct SenderAllocation { rav_request_guard: TokioMutex<()>, unaggregated_receipts_guard: TokioMutex<()>, tap_eip712_domain_separator: Eip712Domain, + sender_account_ref: ActorRef, +} + +pub enum SenderAllocationMsg { + NewReceipt(NewReceiptNotification), + TriggerRAVRequest(RpcReplyPort), + CloseAllocation, +} + +#[async_trait::async_trait] +impl Actor for SenderAllocation { + type Msg = SenderAllocationMsg; + type State = UnaggregatedReceipts; + type Arguments = (); + + async fn pre_start( + &self, + _myself: ActorRef, + _args: Self::Arguments, + ) -> std::result::Result { + Ok(UnaggregatedReceipts::default()) + } + + async fn handle( + &self, + myself: ActorRef, + message: Self::Msg, + state: &mut Self::State, + ) -> std::result::Result<(), ActorProcessingErr> { + match message { + SenderAllocationMsg::NewReceipt(NewReceiptNotification { + id, value: fees, .. + }) => { + if id < state.last_id { + state.last_id = id; + state.value = state.value.checked_add(fees).unwrap_or_else(|| { + // This should never happen, but if it does, we want to know about it. + error!( + "Overflow when adding receipt value {} to total unaggregated fees {} \ + for allocation {} and sender {}. Setting total unaggregated fees to \ + u128::MAX.", + fees, state.value, self.allocation_id, self.sender + ); + u128::MAX + }); + // TODO send a message back to update the unaggregated fees of sender_account + } + } + SenderAllocationMsg::TriggerRAVRequest(reply) => { + self.rav_requester_single().await.map_err(|e| { + anyhow! { + "Error while requesting RAV for sender {} and allocation {}: {}", + self.sender, + self.allocation_id, + e + } + })?; + + // TODO update with the new unaggregated fees + // An error sending means no one is listening anymore (the receiver was dropped), + // so we should shortcut the processing of this message probably! + if !reply.is_closed() { + let _ = reply.send(state.clone()); + } + } + SenderAllocationMsg::CloseAllocation => { + self.rav_requester_single().await.inspect_err(|e| { + error!( + "Error while requesting RAV for sender {} and allocation {}: {}", + self.sender, self.allocation_id, e + ); + })?; + self.mark_rav_final().await.inspect_err(|e| { + error!( + "Error while marking allocation {} as final for sender {}: {}", + self.allocation_id, self.sender, e + ); + })?; + // send back the receipt fees to sender_account + self.sender_account_ref + .cast(SenderAccountMessage::UpdateReceiptFees( + self.allocation_id, + UnaggregatedReceipts { + value: 0, + last_id: 0, + }, + ))?; + myself.stop(None); + } + } + Ok(()) + } } impl SenderAllocation { @@ -64,6 +159,7 @@ impl SenderAllocation { escrow_adapter: EscrowAdapter, tap_eip712_domain_separator: Eip712Domain, sender_aggregator_endpoint: String, + sender_account_ref: ActorRef, ) -> Self { let required_checks: Vec> = vec![ Arc::new(AllocationId::new( @@ -102,6 +198,7 @@ impl SenderAllocation { rav_request_guard: TokioMutex::new(()), unaggregated_receipts_guard: TokioMutex::new(()), tap_eip712_domain_separator, + sender_account_ref, }; sender_allocation @@ -189,7 +286,7 @@ impl SenderAllocation { /// Request a RAV from the sender's TAP aggregator. Only one RAV request will be running at a /// time through the use of an internal guard. - pub async fn rav_requester_single(&self) -> Result<()> { + async fn rav_requester_single(&self) -> Result<()> { // Making extra sure that only one RAV request is running at a time. let _guard = self.rav_request_guard.lock().await; @@ -372,50 +469,6 @@ impl SenderAllocation { Ok(()) } - - /// Safe add the fees to the unaggregated fees value if the receipt_id is greater than the - /// last_id. If the addition would overflow u128, log an error and set the unaggregated fees - /// value to u128::MAX. - /// - /// Returns true if the fees were added, false otherwise. - pub async fn fees_add(&self, fees: u128, receipt_id: u64) -> bool { - // Make sure to pause the handling of receipt notifications while we update the unaggregated - // fees. - let _guard = self.unaggregated_receipts_guard.lock().await; - - let mut fees_added = false; - let mut unaggregated_fees = self.unaggregated_fees.lock().unwrap(); - - if receipt_id > unaggregated_fees.last_id { - *unaggregated_fees = UnaggregatedReceipts { - last_id: receipt_id, - value: unaggregated_fees - .value - .checked_add(fees) - .unwrap_or_else(|| { - // This should never happen, but if it does, we want to know about it. - error!( - "Overflow when adding receipt value {} to total unaggregated fees {} \ - for allocation {} and sender {}. Setting total unaggregated fees to \ - u128::MAX.", - fees, unaggregated_fees.value, self.allocation_id, self.sender - ); - u128::MAX - }), - }; - fees_added = true; - } - - fees_added - } - - pub fn get_unaggregated_fees(&self) -> UnaggregatedReceipts { - self.unaggregated_fees.lock().unwrap().clone() - } - - pub fn get_allocation_id(&self) -> Address { - self.allocation_id - } } #[cfg(test)] From c4cbb0134111110e767bca8a2953d0cb7bd50abd Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Tue, 19 Mar 2024 11:18:38 -0300 Subject: [PATCH 06/41] refactor(wip): remove locks and guards from sender_allocation --- tap-agent/src/agent/sender_allocation.rs | 84 ++++++++---------------- 1 file changed, 28 insertions(+), 56 deletions(-) diff --git a/tap-agent/src/agent/sender_allocation.rs b/tap-agent/src/agent/sender_allocation.rs index 1de5ae184..5c0547a8d 100644 --- a/tap-agent/src/agent/sender_allocation.rs +++ b/tap-agent/src/agent/sender_allocation.rs @@ -1,10 +1,7 @@ // Copyright 2023-, GraphOps and Semiotic Labs. // SPDX-License-Identifier: Apache-2.0 -use std::{ - sync::{Arc, Mutex as StdMutex}, - time::Duration, -}; +use std::{sync::Arc, time::Duration}; use alloy_primitives::hex::ToHex; use alloy_sol_types::Eip712Domain; @@ -25,7 +22,6 @@ use tap_core::{ signed_message::EIP712SignedMessage, }; use thegraph::types::Address; -use tokio::sync::Mutex as TokioMutex; use tracing::{error, warn}; use crate::agent::sender_account::SenderAccountMessage; @@ -47,11 +43,8 @@ pub struct SenderAllocation { allocation_id: Address, sender: Address, sender_aggregator_endpoint: String, - unaggregated_fees: Arc>, config: &'static config::Cli, escrow_accounts: Eventual, - rav_request_guard: TokioMutex<()>, - unaggregated_receipts_guard: TokioMutex<()>, tap_eip712_domain_separator: Eip712Domain, sender_account_ref: ActorRef, } @@ -73,7 +66,22 @@ impl Actor for SenderAllocation { _myself: ActorRef, _args: Self::Arguments, ) -> std::result::Result { - Ok(UnaggregatedReceipts::default()) + let unaggregated_fees = self.calculate_unaggregated_fee().await?; + Ok(unaggregated_fees) + } + + async fn post_stop( + &self, + _myself: ActorRef, + _state: &mut Self::State, + ) -> std::result::Result<(), ActorProcessingErr> { + // cleanup receipt fees for sender + self.sender_account_ref + .cast(SenderAccountMessage::UpdateReceiptFees( + self.allocation_id, + UnaggregatedReceipts::default(), + ))?; + Ok(()) } async fn handle( @@ -98,7 +106,11 @@ impl Actor for SenderAllocation { ); u128::MAX }); - // TODO send a message back to update the unaggregated fees of sender_account + self.sender_account_ref + .cast(SenderAccountMessage::UpdateReceiptFees( + self.allocation_id, + state.clone(), + ))?; } } SenderAllocationMsg::TriggerRAVRequest(reply) => { @@ -110,10 +122,7 @@ impl Actor for SenderAllocation { e } })?; - - // TODO update with the new unaggregated fees - // An error sending means no one is listening anymore (the receiver was dropped), - // so we should shortcut the processing of this message probably! + *state = self.calculate_unaggregated_fee().await?; if !reply.is_closed() { let _ = reply.send(state.clone()); } @@ -131,15 +140,6 @@ impl Actor for SenderAllocation { self.allocation_id, self.sender, e ); })?; - // send back the receipt fees to sender_account - self.sender_account_ref - .cast(SenderAccountMessage::UpdateReceiptFees( - self.allocation_id, - UnaggregatedReceipts { - value: 0, - last_id: 0, - }, - ))?; myself.stop(None); } } @@ -186,42 +186,22 @@ impl SenderAllocation { Checks::new(required_checks), ); - let sender_allocation = Self { + Self { pgpool, tap_manager, allocation_id, sender, sender_aggregator_endpoint, - unaggregated_fees: Arc::new(StdMutex::new(UnaggregatedReceipts::default())), config, escrow_accounts, - rav_request_guard: TokioMutex::new(()), - unaggregated_receipts_guard: TokioMutex::new(()), tap_eip712_domain_separator, sender_account_ref, - }; - - sender_allocation - .update_unaggregated_fees() - .await - .map_err(|e| { - error!( - "Error while updating unaggregated fees for allocation {}: {}", - allocation_id, e - ) - }) - .ok(); - - sender_allocation + } } /// Delete obsolete receipts in the DB w.r.t. the last RAV in DB, then update the tap manager /// with the latest unaggregated fees from the database. - async fn update_unaggregated_fees(&self) -> Result<()> { - // Make sure to pause the handling of receipt notifications while we update the unaggregated - // fees. - let _guard = self.unaggregated_receipts_guard.lock().await; - + async fn calculate_unaggregated_fee(&self) -> Result { self.tap_manager.remove_obsolete_receipts().await?; let signers = signers_trimmed(&self.escrow_accounts, self.sender).await?; @@ -270,26 +250,19 @@ impl SenderAllocation { "Exactly one of SUM(value) and MAX(id) is null. This should not happen." ); - *self.unaggregated_fees.lock().unwrap() = UnaggregatedReceipts { + Ok(UnaggregatedReceipts { last_id: res.max.unwrap_or(0).try_into()?, value: res .sum .unwrap_or(BigDecimal::from(0)) .to_string() .parse::()?, - }; - - // TODO: check if we need to run a RAV request here. - - Ok(()) + }) } /// Request a RAV from the sender's TAP aggregator. Only one RAV request will be running at a /// time through the use of an internal guard. async fn rav_requester_single(&self) -> Result<()> { - // Making extra sure that only one RAV request is running at a time. - let _guard = self.rav_request_guard.lock().await; - let RAVRequest { valid_receipts, previous_rav, @@ -374,7 +347,6 @@ impl SenderAllocation { anyhow::bail!("Error while verifying and storing RAV: {:?}", e); } } - Self::update_unaggregated_fees(self).await?; Ok(()) } From 266c5cbed36b6bd462b0a90483b8842dff653d78 Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Tue, 19 Mar 2024 11:18:52 -0300 Subject: [PATCH 07/41] style: cargo fmt --- tap-agent/src/agent.rs | 2 +- tap-agent/src/agent/sender_account.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tap-agent/src/agent.rs b/tap-agent/src/agent.rs index 61716d4e7..b51e74c9b 100644 --- a/tap-agent/src/agent.rs +++ b/tap-agent/src/agent.rs @@ -11,11 +11,11 @@ use indexer_common::prelude::{ use crate::{aggregator_endpoints, config, database}; use sender_accounts_manager::SenderAccountsManager; +mod allocation_id_tracker; mod sender_account; pub mod sender_accounts_manager; mod sender_allocation; mod unaggregated_receipts; -mod allocation_id_tracker; pub async fn start_agent(config: &'static config::Cli) -> SenderAccountsManager { let pgpool = database::connect(&config.postgres).await; diff --git a/tap-agent/src/agent/sender_account.rs b/tap-agent/src/agent/sender_account.rs index 571a98d13..dc390cf24 100644 --- a/tap-agent/src/agent/sender_account.rs +++ b/tap-agent/src/agent/sender_account.rs @@ -92,7 +92,7 @@ impl Actor for SenderAccount { myself.clone(), ) .await; - let sender_id = self.sender.clone(); + let sender_id = self.sender; let (_actor, _handle) = SenderAllocation::spawn_linked( Some(format!("{sender_id}:{allocation_id}")), From b015eff3f45aa225807e282349c6955ba9612370 Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Tue, 19 Mar 2024 11:50:18 -0300 Subject: [PATCH 08/41] refactor: add sender allocation creation --- tap-agent/src/agent/sender_account.rs | 125 ++++++++++++------ .../src/agent/sender_accounts_manager.rs | 7 +- 2 files changed, 89 insertions(+), 43 deletions(-) diff --git a/tap-agent/src/agent/sender_account.rs b/tap-agent/src/agent/sender_account.rs index dc390cf24..7b53443ea 100644 --- a/tap-agent/src/agent/sender_account.rs +++ b/tap-agent/src/agent/sender_account.rs @@ -1,15 +1,17 @@ // Copyright 2023-, GraphOps and Semiotic Labs. // SPDX-License-Identifier: Apache-2.0 -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use alloy_sol_types::Eip712Domain; use anyhow::Result; -use eventuals::Eventual; +use eventuals::{Eventual, EventualExt, PipeHandle}; +use indexer_common::allocations::Allocation; use indexer_common::{escrow_accounts::EscrowAccounts, prelude::SubgraphClient}; use ractor::{call, Actor, ActorProcessingErr, ActorRef}; use sqlx::PgPool; use thegraph::types::Address; +use tracing::error; use super::sender_allocation::SenderAllocation; use crate::agent::allocation_id_tracker::AllocationIdTracker; @@ -22,6 +24,7 @@ use crate::{ pub enum SenderAccountMessage { CreateSenderAllocation(Address), + UpdateAllocationIds(HashSet
), RemoveSenderAccount, UpdateReceiptFees(Address, UnaggregatedReceipts), } @@ -36,7 +39,10 @@ pub enum SenderAccountMessage { /// certain threshold. /// - Requesting the last RAV from the sender's TAP aggregator for all EOL allocations. pub struct SenderAccount { + //Eventuals escrow_accounts: Eventual, + indexer_allocations: Eventual>, + escrow_subgraph: &'static SubgraphClient, escrow_adapter: EscrowAdapter, tap_eip712_domain_separator: Eip712Domain, @@ -46,21 +52,51 @@ pub struct SenderAccount { sender_aggregator_endpoint: String, } +pub struct State { + allocation_id_tracker: AllocationIdTracker, + allocation_ids: HashSet
, + _indexer_allocations_handle: PipeHandle, +} + #[async_trait::async_trait] impl Actor for SenderAccount { type Msg = SenderAccountMessage; - type State = AllocationIdTracker; + type State = State; type Arguments = HashSet
; async fn pre_start( &self, - _myself: ActorRef, - _args: Self::Arguments, + myself: ActorRef, + allocation_ids: Self::Arguments, ) -> std::result::Result { - // todo look for allocations and create sender_allocations + let clone = myself.clone(); + let _indexer_allocations_handle = + self.indexer_allocations + .clone() + .pipe_async(move |escrow_accounts| { + let myself = clone.clone(); + async move { + // Update the allocation_ids + myself + .cast(SenderAccountMessage::UpdateAllocationIds( + escrow_accounts.keys().cloned().into_iter().collect(), + )) + .unwrap_or_else(|e| { + error!("Error while updating allocation_ids: {:?}", e); + }); + } + }); - // todo use the arguments to spawn the initial sender allocations - Ok(AllocationIdTracker::new()) + for allocation_id in &allocation_ids { + // Create a sender allocation for each allocation + myself.cast(SenderAccountMessage::CreateSenderAllocation(*allocation_id))?; + } + + Ok(State { + allocation_id_tracker: AllocationIdTracker::new(), + allocation_ids, + _indexer_allocations_handle, + }) } async fn handle( @@ -72,10 +108,11 @@ impl Actor for SenderAccount { match message { SenderAccountMessage::RemoveSenderAccount => myself.stop(None), SenderAccountMessage::UpdateReceiptFees(allocation_id, unaggregated_fees) => { - state.add_or_update(allocation_id, unaggregated_fees.value); + let tracker = &mut state.allocation_id_tracker; + tracker.add_or_update(allocation_id, unaggregated_fees.value); - if state.get_total_fee() >= self.config.tap.rav_request_trigger_value.into() { - self.rav_requester_single(state).await?; + if tracker.get_total_fee() >= self.config.tap.rav_request_trigger_value.into() { + self.rav_requester_single(tracker).await?; } } SenderAccountMessage::CreateSenderAllocation(allocation_id) => { @@ -102,6 +139,24 @@ impl Actor for SenderAccount { ) .await?; } + SenderAccountMessage::UpdateAllocationIds(allocation_ids) => { + // Create new sender allocations + for allocation_id in allocation_ids.difference(&state.allocation_ids) { + myself.cast(SenderAccountMessage::CreateSenderAllocation(*allocation_id))?; + } + + // Remove sender allocations + let sender = self.sender; + for allocation_id in state.allocation_ids.difference(&allocation_ids) { + if let Some(sender_handle) = ActorRef::::where_is(format!( + "{sender}:{allocation_id}" + )) { + sender_handle.cast(SenderAllocationMsg::CloseAllocation)?; + } + } + + state.allocation_ids = allocation_ids; + } } Ok(()) } @@ -114,6 +169,7 @@ impl SenderAccount { pgpool: PgPool, sender_id: Address, escrow_accounts: Eventual, + indexer_allocations: Eventual>, escrow_subgraph: &'static SubgraphClient, tap_eip712_domain_separator: Eip712Domain, sender_aggregator_endpoint: String, @@ -122,6 +178,7 @@ impl SenderAccount { Self { escrow_accounts, + indexer_allocations, escrow_subgraph, escrow_adapter, tap_eip712_domain_separator, @@ -156,11 +213,14 @@ impl SenderAccount { #[cfg(test)] mod tests { + use crate::agent::sender_accounts_manager::NewReceiptNotification; use alloy_primitives::hex::ToHex; use bigdecimal::{num_bigint::ToBigInt, ToPrimitive}; use indexer_common::subgraph_client::DeploymentDetails; use serde_json::json; + use std::collections::HashMap; use std::str::FromStr; + use std::sync::Arc; use tap_aggregator::server::run_server; use tap_core::{rav::ReceiptAggregateVoucher, signed_message::EIP712SignedMessage}; use wiremock::{ @@ -180,8 +240,7 @@ mod tests { // To help with testing from other modules. impl SenderAccount { pub fn _tests_get_allocations_active(&self) -> HashMap> { - self.inner - .allocations + self.allocations .lock() .unwrap() .iter() @@ -196,8 +255,7 @@ mod tests { } pub fn _tests_get_allocations_ineligible(&self) -> HashMap> { - self.inner - .allocations + self.allocations .lock() .unwrap() .iter() @@ -287,7 +345,6 @@ mod tests { // Check that the allocation's unaggregated fees are correct. assert_eq!( sender - .inner .allocations .lock() .unwrap() @@ -302,7 +359,7 @@ mod tests { // Check that the sender's unaggregated fees are correct. assert_eq!( - sender.inner.unaggregated_fees.lock().unwrap().value, + sender.unaggregated_fees.lock().unwrap().value, expected_unaggregated_fees ); @@ -323,7 +380,6 @@ mod tests { // Check that the allocation's unaggregated fees have *not* increased. assert_eq!( sender - .inner .allocations .lock() .unwrap() @@ -338,7 +394,7 @@ mod tests { // Check that the unaggregated fees have *not* increased. assert_eq!( - sender.inner.unaggregated_fees.lock().unwrap().value, + sender.unaggregated_fees.lock().unwrap().value, expected_unaggregated_fees ); @@ -358,7 +414,6 @@ mod tests { // Check that the allocation's unaggregated fees are correct. assert_eq!( sender - .inner .allocations .lock() .unwrap() @@ -373,7 +428,7 @@ mod tests { // Check that the unaggregated fees are correct. assert_eq!( - sender.inner.unaggregated_fees.lock().unwrap().value, + sender.unaggregated_fees.lock().unwrap().value, expected_unaggregated_fees ); @@ -392,7 +447,6 @@ mod tests { // Check that the allocation's unaggregated fees have *not* increased. assert_eq!( sender - .inner .allocations .lock() .unwrap() @@ -407,7 +461,7 @@ mod tests { // Check that the unaggregated fees have *not* increased. assert_eq!( - sender.inner.unaggregated_fees.lock().unwrap().value, + sender.unaggregated_fees.lock().unwrap().value, expected_unaggregated_fees ); } @@ -482,7 +536,7 @@ mod tests { // Wait for the RAV requester to finish. for _ in 0..100 { - if sender_account.inner.unaggregated_fees.lock().unwrap().value < trigger_value { + if sender_account.unaggregated_fees.lock().unwrap().value < trigger_value { break; } tokio::time::sleep(std::time::Duration::from_millis(100)).await; @@ -525,7 +579,6 @@ mod tests { // Check that the allocation's unaggregated fees value is reduced. assert!( sender_account - .inner .allocations .lock() .unwrap() @@ -539,10 +592,10 @@ mod tests { ); // Check that the sender's unaggregated fees value is reduced. - assert!(sender_account.inner.unaggregated_fees.lock().unwrap().value <= trigger_value); + assert!(sender_account.unaggregated_fees.lock().unwrap().value <= trigger_value); // Reset the total value and trigger value. - total_value = sender_account.inner.unaggregated_fees.lock().unwrap().value; + total_value = sender_account.unaggregated_fees.lock().unwrap().value; trigger_value = 0; // Add more receipts @@ -573,7 +626,7 @@ mod tests { // Wait for the RAV requester to finish. for _ in 0..100 { - if sender_account.inner.unaggregated_fees.lock().unwrap().value < trigger_value { + if sender_account.unaggregated_fees.lock().unwrap().value < trigger_value { break; } tokio::time::sleep(std::time::Duration::from_millis(100)).await; @@ -617,7 +670,6 @@ mod tests { // Check that the allocation's unaggregated fees value is reduced. assert!( sender_account - .inner .allocations .lock() .unwrap() @@ -631,7 +683,7 @@ mod tests { ); // Check that the unaggregated fees value is reduced. - assert!(sender_account.inner.unaggregated_fees.lock().unwrap().value <= trigger_value); + assert!(sender_account.unaggregated_fees.lock().unwrap().value <= trigger_value); // Stop the TAP aggregator server. handle.stop().unwrap(); @@ -654,13 +706,7 @@ mod tests { for i in 0..iterations { let value = (i + 10) as u128; - let id = sender_account - .inner - .unaggregated_fees - .lock() - .unwrap() - .last_id - + 1; + let id = sender_account.unaggregated_fees.lock().unwrap().last_id + 1; sender_account .handle_new_receipt_notification(NewReceiptNotification { @@ -677,7 +723,6 @@ mod tests { assert_eq!( sender_account - .inner .allocations .lock() .unwrap() @@ -704,14 +749,14 @@ mod tests { let total_value_2 = add_receipts(*ALLOCATION_ID_2, 8).await; // Get the heaviest allocation. - let heaviest_allocation = sender_account.inner.get_heaviest_allocation().unwrap(); + let heaviest_allocation = sender_account.get_heaviest_allocation().unwrap(); // Check that the heaviest allocation is correct. assert_eq!(heaviest_allocation.get_allocation_id(), *ALLOCATION_ID_1); // Check that the sender's unaggregated fees value is correct. assert_eq!( - sender_account.inner.unaggregated_fees.lock().unwrap().value, + sender_account.unaggregated_fees.lock().unwrap().value, total_value_0 + total_value_1 + total_value_2 ); } diff --git a/tap-agent/src/agent/sender_accounts_manager.rs b/tap-agent/src/agent/sender_accounts_manager.rs index da5fb0714..45f18fff2 100644 --- a/tap-agent/src/agent/sender_accounts_manager.rs +++ b/tap-agent/src/agent/sender_accounts_manager.rs @@ -77,7 +77,7 @@ impl Actor for SenderAccountsManager { self.escrow_accounts.clone(), )); let clone = myself.clone(); - let eligible_allocations_senders_pipe = + let _eligible_allocations_senders_pipe = self.escrow_accounts .clone() .pipe_async(move |escrow_accounts| { @@ -104,12 +104,12 @@ impl Actor for SenderAccountsManager { Ok(State { sender_ids: HashSet::new(), new_receipts_watcher_handle, - _eligible_allocations_senders_pipe: eligible_allocations_senders_pipe, + _eligible_allocations_senders_pipe, }) } async fn post_stop( &self, - _myself: ActorRef, + _: ActorRef, state: &mut Self::State, ) -> std::result::Result<(), ActorProcessingErr> { // Abort the notification watcher on drop. Otherwise it may panic because the PgPool could @@ -282,6 +282,7 @@ impl SenderAccountsManager { self.pgpool.clone(), *sender_id, self.escrow_accounts.clone(), + self.indexer_allocations.clone(), self.escrow_subgraph, self.tap_eip712_domain_separator.clone(), self.sender_aggregator_endpoints From 7762c68b4dc75edceed192e5871088a4560c19c0 Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Tue, 19 Mar 2024 21:12:04 -0300 Subject: [PATCH 09/41] test: update sender accounts manager tests to actors --- .../src/agent/sender_accounts_manager.rs | 48 +++++++++---------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/tap-agent/src/agent/sender_accounts_manager.rs b/tap-agent/src/agent/sender_accounts_manager.rs index 45f18fff2..733268156 100644 --- a/tap-agent/src/agent/sender_accounts_manager.rs +++ b/tap-agent/src/agent/sender_accounts_manager.rs @@ -32,7 +32,7 @@ pub struct NewReceiptNotification { pub struct SenderAccountsManager { config: &'static config::Cli, pgpool: PgPool, - indexer_allocations: Eventual>, + indexer_allocations: Eventual>, escrow_accounts: Eventual, escrow_subgraph: &'static SubgraphClient, tap_eip712_domain_separator: Eip712Domain, @@ -167,6 +167,9 @@ impl SenderAccountsManager { tap_eip712_domain_separator: Eip712Domain, sender_aggregator_endpoints: HashMap, ) -> Self { + let indexer_allocations = indexer_allocations.map(|allocations| async move { + allocations.keys().cloned().collect::>() + }); Self { config, pgpool, @@ -364,6 +367,7 @@ mod tests { prelude::{AllocationStatus, SubgraphDeployment}, subgraph_client::DeploymentDetails, }; + use ractor::ActorStatus; use serde_json::json; use thegraph::types::DeploymentId; use wiremock::{ @@ -426,6 +430,9 @@ mod tests { HashMap::from([(SENDER.1, String::from("http://localhost:8000"))]), ) .await; + SenderAccountsManager::spawn(None, sender_account, ()) + .await + .unwrap(); let allocation_id = Address::from_str("0xdd975e30aafebb143e54d215db8a3e8fd916a701").unwrap(); @@ -466,12 +473,14 @@ mod tests { tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; // Check that the SenderAccount was created. - assert!(sender_account - .inner - .sender_accounts - .lock() - .unwrap() - .contains_key(&SENDER.1)); + let sender_account = + ActorRef::::where_is(SENDER.1.to_string()).unwrap(); + assert_eq!(sender_account.get_status(), ActorStatus::Running); + + let sender_allocation_id = format!("{}:{}", SENDER.1, allocation_id); + let sender_allocation = + ActorRef::::where_is(sender_allocation_id).unwrap(); + assert_eq!(sender_allocation.get_status(), ActorStatus::Running); // Remove the escrow account from the escrow_accounts Eventual. escrow_accounts_writer.write(EscrowAccounts::default()); @@ -480,23 +489,12 @@ mod tests { tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; // Check that the Sender's allocation moved from active to ineligible. - assert!(sender_account - .inner - .sender_accounts - .lock() - .unwrap() - .get(&SENDER.1) - .unwrap() - ._tests_get_allocations_active() - .is_empty()); - assert!(sender_account - .inner - .sender_accounts - .lock() - .unwrap() - .get(&SENDER.1) - .unwrap() - ._tests_get_allocations_ineligible() - .contains_key(&allocation_id)); + + assert_eq!(sender_account.get_status(), ActorStatus::Stopped); + assert_eq!(sender_allocation.get_status(), ActorStatus::Stopped); + + let sender_account = ActorRef::::where_is(SENDER.1.to_string()); + + assert!(sender_account.is_none()); } } From 293aedec726b867e91306b33f9c74ced3a2c3b79 Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Tue, 19 Mar 2024 21:13:52 -0300 Subject: [PATCH 10/41] test: wip update tests for sender account --- tap-agent/src/agent/allocation_id_tracker.rs | 2 +- tap-agent/src/agent/sender_account.rs | 445 +++++++++---------- tap-agent/src/agent/sender_allocation.rs | 392 ++++++++-------- 3 files changed, 421 insertions(+), 418 deletions(-) diff --git a/tap-agent/src/agent/allocation_id_tracker.rs b/tap-agent/src/agent/allocation_id_tracker.rs index 802606a12..8ee702329 100644 --- a/tap-agent/src/agent/allocation_id_tracker.rs +++ b/tap-agent/src/agent/allocation_id_tracker.rs @@ -1,7 +1,7 @@ use alloy_primitives::Address; use std::collections::{BTreeMap, HashMap}; -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct AllocationIdTracker { id_to_fee: HashMap, fee_to_count: BTreeMap, diff --git a/tap-agent/src/agent/sender_account.rs b/tap-agent/src/agent/sender_account.rs index 7b53443ea..228b3b5eb 100644 --- a/tap-agent/src/agent/sender_account.rs +++ b/tap-agent/src/agent/sender_account.rs @@ -8,7 +8,7 @@ use anyhow::Result; use eventuals::{Eventual, EventualExt, PipeHandle}; use indexer_common::allocations::Allocation; use indexer_common::{escrow_accounts::EscrowAccounts, prelude::SubgraphClient}; -use ractor::{call, Actor, ActorProcessingErr, ActorRef}; +use ractor::{call, Actor, ActorProcessingErr, ActorRef, RpcReplyPort}; use sqlx::PgPool; use thegraph::types::Address; use tracing::error; @@ -27,6 +27,7 @@ pub enum SenderAccountMessage { UpdateAllocationIds(HashSet
), RemoveSenderAccount, UpdateReceiptFees(Address, UnaggregatedReceipts), + GetAllocationTracker(RpcReplyPort), } /// A SenderAccount manages the receipts accounting between the indexer and the sender across @@ -41,7 +42,7 @@ pub enum SenderAccountMessage { pub struct SenderAccount { //Eventuals escrow_accounts: Eventual, - indexer_allocations: Eventual>, + indexer_allocations: Eventual>, escrow_subgraph: &'static SubgraphClient, escrow_adapter: EscrowAdapter, @@ -73,14 +74,12 @@ impl Actor for SenderAccount { let _indexer_allocations_handle = self.indexer_allocations .clone() - .pipe_async(move |escrow_accounts| { + .pipe_async(move |allocation_ids| { let myself = clone.clone(); async move { // Update the allocation_ids myself - .cast(SenderAccountMessage::UpdateAllocationIds( - escrow_accounts.keys().cloned().into_iter().collect(), - )) + .cast(SenderAccountMessage::UpdateAllocationIds(allocation_ids)) .unwrap_or_else(|e| { error!("Error while updating allocation_ids: {:?}", e); }); @@ -157,6 +156,11 @@ impl Actor for SenderAccount { state.allocation_ids = allocation_ids; } + SenderAccountMessage::GetAllocationTracker(reply) => { + if !reply.is_closed() { + let _ = reply.send(state.allocation_id_tracker.clone()); + } + } } Ok(()) } @@ -169,7 +173,7 @@ impl SenderAccount { pgpool: PgPool, sender_id: Address, escrow_accounts: Eventual, - indexer_allocations: Eventual>, + indexer_allocations: Eventual>, escrow_subgraph: &'static SubgraphClient, tap_eip712_domain_separator: Eip712Domain, sender_aggregator_endpoint: String, @@ -216,13 +220,17 @@ mod tests { use crate::agent::sender_accounts_manager::NewReceiptNotification; use alloy_primitives::hex::ToHex; use bigdecimal::{num_bigint::ToBigInt, ToPrimitive}; + use ethereum_types::U256; + use indexer_common::allocations::{AllocationStatus, SubgraphDeployment}; use indexer_common::subgraph_client::DeploymentDetails; use serde_json::json; use std::collections::HashMap; use std::str::FromStr; use std::sync::Arc; + use std::time::Duration; use tap_aggregator::server::run_server; use tap_core::{rav::ReceiptAggregateVoucher, signed_message::EIP712SignedMessage}; + use thegraph::types::DeploymentId; use wiremock::{ matchers::{body_string_contains, method}, Mock, MockServer, ResponseTemplate, @@ -236,52 +244,23 @@ mod tests { use super::*; const DUMMY_URL: &str = "http://localhost:1234"; + const VALUE_PER_RECEIPT: u64 = 100; + const TRIGGER_VALUE: u64 = 500; // To help with testing from other modules. - impl SenderAccount { - pub fn _tests_get_allocations_active(&self) -> HashMap> { - self.allocations - .lock() - .unwrap() - .iter() - .filter_map(|(k, v)| { - if let AllocationState::Active(a) = v { - Some((*k, a.clone())) - } else { - None - } - }) - .collect() - } - - pub fn _tests_get_allocations_ineligible(&self) -> HashMap> { - self.allocations - .lock() - .unwrap() - .iter() - .filter_map(|(k, v)| { - if let AllocationState::Ineligible(a) = v { - Some((*k, a.clone())) - } else { - None - } - }) - .collect() - } - } async fn create_sender_with_allocations( pgpool: PgPool, sender_aggregator_endpoint: String, escrow_subgraph_endpoint: &str, - ) -> SenderAccount { + ) -> ActorRef { let config = Box::leak(Box::new(config::Cli { config: None, ethereum: config::Ethereum { indexer_address: INDEXER.1, }, tap: config::Tap { - rav_request_trigger_value: 100, + rav_request_trigger_value: TRIGGER_VALUE, rav_request_timestamp_buffer_ms: 1, rav_request_timeout_secs: 5, ..Default::default() @@ -300,24 +279,33 @@ mod tests { HashMap::from([(SENDER.1, vec![SIGNER.1])]), )); + let allocation_id = + Address::from_str("0xdd975e30aafebb143e54d215db8a3e8fd916a701").unwrap(); + + let indexer_allocations = Eventual::from_value(HashSet::from([ + *ALLOCATION_ID_0, + *ALLOCATION_ID_1, + *ALLOCATION_ID_2, + ])); + let sender = SenderAccount::new( config, pgpool, SENDER.1, escrow_accounts_eventual, + indexer_allocations, escrow_subgraph, TAP_EIP712_DOMAIN_SEPARATOR.clone(), sender_aggregator_endpoint, ); - sender - .update_allocations(HashSet::from([ - *ALLOCATION_ID_0, - *ALLOCATION_ID_1, - *ALLOCATION_ID_2, - ])) - .await; - sender.recompute_unaggregated_fees().await; + let (sender, _handle) = + SenderAccount::spawn(Some(SENDER.1.to_string()), sender, HashSet::new()) + .await + .unwrap(); + + // await for the allocations to be created + ractor::concurrency::sleep(Duration::from_millis(100)).await; sender } @@ -342,24 +330,26 @@ mod tests { let sender = create_sender_with_allocations(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL).await; - // Check that the allocation's unaggregated fees are correct. + let allocation = ActorRef::::where_is(format!( + "{sender}:{allocation_id}", + sender = SENDER.1, + allocation_id = ALLOCATION_ID_0.to_string() + )) + .unwrap(); + + // Check that the sender's unaggregated fees are correct. + let allocation_tracker = call!(sender, SenderAccountMessage::GetAllocationTracker).unwrap(); assert_eq!( - sender - .allocations - .lock() - .unwrap() - .get(&*ALLOCATION_ID_0) - .unwrap() - .as_active() - .unwrap() - .get_unaggregated_fees() - .value, + allocation_tracker.get_total_fee(), expected_unaggregated_fees ); - // Check that the sender's unaggregated fees are correct. + let allocation_unaggregated_fees = + call!(allocation, SenderAllocationMsg::GetUnaggregatedReceipts).unwrap(); + + // Check that the allocation's unaggregated fees are correct. assert_eq!( - sender.unaggregated_fees.lock().unwrap().value, + allocation_unaggregated_fees.value, expected_unaggregated_fees ); @@ -373,28 +363,24 @@ mod tests { timestamp_ns: 19, value: 19, }; - sender - .handle_new_receipt_notification(new_receipt_notification) - .await; + allocation + .cast(SenderAllocationMsg::NewReceipt(new_receipt_notification)) + .unwrap(); + ractor::concurrency::sleep(Duration::from_millis(10)).await; + + let allocation_unaggregated_fees = + call!(allocation, SenderAllocationMsg::GetUnaggregatedReceipts).unwrap(); // Check that the allocation's unaggregated fees have *not* increased. assert_eq!( - sender - .allocations - .lock() - .unwrap() - .get(&*ALLOCATION_ID_0) - .unwrap() - .as_active() - .unwrap() - .get_unaggregated_fees() - .value, + allocation_unaggregated_fees.value, expected_unaggregated_fees ); // Check that the unaggregated fees have *not* increased. + let allocation_tracker = call!(sender, SenderAccountMessage::GetAllocationTracker).unwrap(); assert_eq!( - sender.unaggregated_fees.lock().unwrap().value, + allocation_tracker.get_total_fee(), expected_unaggregated_fees ); @@ -406,29 +392,26 @@ mod tests { timestamp_ns: 20, value: 20, }; - sender - .handle_new_receipt_notification(new_receipt_notification) - .await; + allocation + .cast(SenderAllocationMsg::NewReceipt(new_receipt_notification)) + .unwrap(); + ractor::concurrency::sleep(Duration::from_millis(10)).await; + expected_unaggregated_fees += 20; + let allocation_unaggregated_fees = + call!(allocation, SenderAllocationMsg::GetUnaggregatedReceipts).unwrap(); + // Check that the allocation's unaggregated fees are correct. assert_eq!( - sender - .allocations - .lock() - .unwrap() - .get(&*ALLOCATION_ID_0) - .unwrap() - .as_active() - .unwrap() - .get_unaggregated_fees() - .value, + allocation_unaggregated_fees.value, expected_unaggregated_fees ); - // Check that the unaggregated fees are correct. + // Check that the sender's unaggregated fees are correct. + let allocation_tracker = call!(sender, SenderAccountMessage::GetAllocationTracker).unwrap(); assert_eq!( - sender.unaggregated_fees.lock().unwrap().value, + allocation_tracker.get_total_fee(), expected_unaggregated_fees ); @@ -440,28 +423,25 @@ mod tests { timestamp_ns: 19, value: 19, }; - sender - .handle_new_receipt_notification(new_receipt_notification) - .await; + allocation + .cast(SenderAllocationMsg::NewReceipt(new_receipt_notification)) + .unwrap(); + + ractor::concurrency::sleep(Duration::from_millis(10)).await; + + let allocation_unaggregated_fees = + call!(allocation, SenderAllocationMsg::GetUnaggregatedReceipts).unwrap(); // Check that the allocation's unaggregated fees have *not* increased. assert_eq!( - sender - .allocations - .lock() - .unwrap() - .get(&*ALLOCATION_ID_0) - .unwrap() - .as_active() - .unwrap() - .get_unaggregated_fees() - .value, + allocation_unaggregated_fees.value, expected_unaggregated_fees ); // Check that the unaggregated fees have *not* increased. + let allocation_tracker = call!(sender, SenderAccountMessage::GetAllocationTracker).unwrap(); assert_eq!( - sender.unaggregated_fees.lock().unwrap().value, + allocation_tracker.get_total_fee(), expected_unaggregated_fees ); } @@ -504,39 +484,53 @@ mod tests { ) .await; + let allocation = ActorRef::::where_is(format!( + "{sender}:{allocation_id}", + sender = SENDER.1, + allocation_id = ALLOCATION_ID_0.to_string() + )) + .unwrap(); + // Add receipts to the database and call the `handle_new_receipt_notification` method // correspondingly. let mut total_value = 0; let mut trigger_value = 0; - for i in 0..10 { + for i in 1..=10 { // These values should be enough to trigger a RAV request at i == 7 since we set the // `rav_request_trigger_value` to 100. - let value = (i + 10) as u128; + let value = (i + VALUE_PER_RECEIPT) as u128; let receipt = create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i + 1, value).await; store_receipt(&pgpool, receipt.signed_receipt()) .await .unwrap(); - sender_account - .handle_new_receipt_notification(NewReceiptNotification { - allocation_id: *ALLOCATION_ID_0, - signer_address: SIGNER.1, - id: i, - timestamp_ns: i + 1, - value, - }) - .await; + let new_receipt_notification = NewReceiptNotification { + allocation_id: *ALLOCATION_ID_0, + signer_address: SIGNER.1, + id: i, + timestamp_ns: i + 1 , + value, + }; + allocation + .cast(SenderAllocationMsg::NewReceipt(new_receipt_notification)) + .unwrap(); + + ractor::concurrency::sleep(Duration::from_millis(100)).await; total_value += value; - if total_value >= 100 && trigger_value == 0 { + if total_value >= TRIGGER_VALUE as u128 && trigger_value == 0 { trigger_value = total_value; } } + ractor::concurrency::sleep(Duration::from_millis(10)).await; + // Wait for the RAV requester to finish. for _ in 0..100 { - if sender_account.unaggregated_fees.lock().unwrap().value < trigger_value { + let allocation_tracker = + call!(sender_account, SenderAccountMessage::GetAllocationTracker).unwrap(); + if allocation_tracker.get_total_fee() < trigger_value { break; } tokio::time::sleep(std::time::Duration::from_millis(100)).await; @@ -577,30 +571,25 @@ mod tests { assert!(latest_rav.message.valueAggregate >= trigger_value); // Check that the allocation's unaggregated fees value is reduced. - assert!( - sender_account - .allocations - .lock() - .unwrap() - .get(&*ALLOCATION_ID_0) - .unwrap() - .as_active() - .unwrap() - .get_unaggregated_fees() - .value - <= trigger_value - ); + + let allocation_unaggregated_fees = + call!(allocation, SenderAllocationMsg::GetUnaggregatedReceipts).unwrap(); + + // Check that the allocation's unaggregated fees have *not* increased. + assert!(allocation_unaggregated_fees.value <= trigger_value); // Check that the sender's unaggregated fees value is reduced. - assert!(sender_account.unaggregated_fees.lock().unwrap().value <= trigger_value); + let allocation_tracker = + call!(sender_account, SenderAccountMessage::GetAllocationTracker).unwrap(); + assert!(allocation_tracker.get_total_fee() <= trigger_value); // Reset the total value and trigger value. - total_value = sender_account.unaggregated_fees.lock().unwrap().value; + total_value = allocation_tracker.get_total_fee(); trigger_value = 0; // Add more receipts for i in 10..20 { - let value = (i + 10) as u128; + let value = (i + VALUE_PER_RECEIPT) as u128; let receipt = create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i + 1, i.into()).await; @@ -608,25 +597,30 @@ mod tests { .await .unwrap(); - sender_account - .handle_new_receipt_notification(NewReceiptNotification { - allocation_id: *ALLOCATION_ID_0, - signer_address: SIGNER.1, - id: i, - timestamp_ns: i + 1, - value, - }) - .await; + let new_receipt_notification = NewReceiptNotification { + allocation_id: *ALLOCATION_ID_0, + signer_address: SIGNER.1, + id: i, + timestamp_ns: i + 1, + value, + }; + allocation + .cast(SenderAllocationMsg::NewReceipt(new_receipt_notification)) + .unwrap(); + + ractor::concurrency::sleep(Duration::from_millis(10)).await; total_value += value; - if total_value >= 100 && trigger_value == 0 { + if total_value >= TRIGGER_VALUE as u128 && trigger_value == 0 { trigger_value = total_value; } } // Wait for the RAV requester to finish. for _ in 0..100 { - if sender_account.unaggregated_fees.lock().unwrap().value < trigger_value { + let allocation_tracker = + call!(sender_account, SenderAccountMessage::GetAllocationTracker).unwrap(); + if allocation_tracker.get_total_fee() < trigger_value { break; } tokio::time::sleep(std::time::Duration::from_millis(100)).await; @@ -668,96 +662,91 @@ mod tests { assert!(latest_rav.message.valueAggregate >= trigger_value); // Check that the allocation's unaggregated fees value is reduced. - assert!( - sender_account - .allocations - .lock() - .unwrap() - .get(&*ALLOCATION_ID_0) - .unwrap() - .as_active() - .unwrap() - .get_unaggregated_fees() - .value - <= trigger_value - ); + + let allocation_unaggregated_fees = + call!(allocation, SenderAllocationMsg::GetUnaggregatedReceipts).unwrap(); + + // Check that the allocation's unaggregated fees have *not* increased. + assert!(allocation_unaggregated_fees.value <= trigger_value); // Check that the unaggregated fees value is reduced. - assert!(sender_account.unaggregated_fees.lock().unwrap().value <= trigger_value); + let allocation_tracker = + call!(sender_account, SenderAccountMessage::GetAllocationTracker).unwrap(); + assert!(allocation_tracker.get_total_fee() <= trigger_value); // Stop the TAP aggregator server. handle.stop().unwrap(); handle.stopped().await; } - #[sqlx::test(migrations = "../migrations")] - async fn test_sender_unaggregated_fees(pgpool: PgPool) { - // Create a sender_account. - let sender_account = Arc::new( - create_sender_with_allocations(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL).await, - ); - - // Closure that adds a number of receipts to an allocation. - let add_receipts = |allocation_id: Address, iterations: u64| { - let sender_account = sender_account.clone(); - - async move { - let mut total_value = 0; - for i in 0..iterations { - let value = (i + 10) as u128; - - let id = sender_account.unaggregated_fees.lock().unwrap().last_id + 1; - - sender_account - .handle_new_receipt_notification(NewReceiptNotification { - allocation_id, - signer_address: SIGNER.1, - id, - timestamp_ns: i + 1, - value, - }) - .await; - - total_value += value; - } - - assert_eq!( - sender_account - .allocations - .lock() - .unwrap() - .get(&allocation_id) - .unwrap() - .as_active() - .unwrap() - .get_unaggregated_fees() - .value, - total_value - ); - - total_value - } - }; - - // Add receipts to the database for allocation_0 - let total_value_0 = add_receipts(*ALLOCATION_ID_0, 9).await; - - // Add receipts to the database for allocation_1 - let total_value_1 = add_receipts(*ALLOCATION_ID_1, 10).await; - - // Add receipts to the database for allocation_2 - let total_value_2 = add_receipts(*ALLOCATION_ID_2, 8).await; - - // Get the heaviest allocation. - let heaviest_allocation = sender_account.get_heaviest_allocation().unwrap(); - - // Check that the heaviest allocation is correct. - assert_eq!(heaviest_allocation.get_allocation_id(), *ALLOCATION_ID_1); - - // Check that the sender's unaggregated fees value is correct. - assert_eq!( - sender_account.unaggregated_fees.lock().unwrap().value, - total_value_0 + total_value_1 + total_value_2 - ); - } + // #[sqlx::test(migrations = "../migrations")] + // async fn test_sender_unaggregated_fees(pgpool: PgPool) { + // // Create a sender_account. + // let sender_account = Arc::new( + // create_sender_with_allocations(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL).await, + // ); + // + // // Closure that adds a number of receipts to an allocation. + // let add_receipts = |allocation_id: Address, iterations: u64| { + // let sender_account = sender_account.clone(); + // + // async move { + // let mut total_value = 0; + // for i in 0..iterations { + // let value = (i + 10) as u128; + // + // let id = sender_account.unaggregated_fees.lock().unwrap().last_id + 1; + // + // sender_account + // .handle_new_receipt_notification(NewReceiptNotification { + // allocation_id, + // signer_address: SIGNER.1, + // id, + // timestamp_ns: i + 1, + // value, + // }) + // .await; + // + // total_value += value; + // } + // + // assert_eq!( + // sender_account + // .allocations + // .lock() + // .unwrap() + // .get(&allocation_id) + // .unwrap() + // .as_active() + // .unwrap() + // .get_unaggregated_fees() + // .value, + // total_value + // ); + // + // total_value + // } + // }; + // + // // Add receipts to the database for allocation_0 + // let total_value_0 = add_receipts(*ALLOCATION_ID_0, 9).await; + // + // // Add receipts to the database for allocation_1 + // let total_value_1 = add_receipts(*ALLOCATION_ID_1, 10).await; + // + // // Add receipts to the database for allocation_2 + // let total_value_2 = add_receipts(*ALLOCATION_ID_2, 8).await; + // + // // Get the heaviest allocation. + // let heaviest_allocation = sender_account.get_heaviest_allocation().unwrap(); + // + // // Check that the heaviest allocation is correct. + // assert_eq!(heaviest_allocation.get_allocation_id(), *ALLOCATION_ID_1); + // + // // Check that the sender's unaggregated fees value is correct. + // assert_eq!( + // sender_account.unaggregated_fees.lock().unwrap().value, + // total_value_0 + total_value_1 + total_value_2 + // ); + // } } diff --git a/tap-agent/src/agent/sender_allocation.rs b/tap-agent/src/agent/sender_allocation.rs index 5c0547a8d..3828505d8 100644 --- a/tap-agent/src/agent/sender_allocation.rs +++ b/tap-agent/src/agent/sender_allocation.rs @@ -24,6 +24,7 @@ use tap_core::{ use thegraph::types::Address; use tracing::{error, warn}; +use crate::agent::allocation_id_tracker::AllocationIdTracker; use crate::agent::sender_account::SenderAccountMessage; use crate::agent::sender_accounts_manager::NewReceiptNotification; use crate::agent::unaggregated_receipts::UnaggregatedReceipts; @@ -52,6 +53,7 @@ pub struct SenderAllocation { pub enum SenderAllocationMsg { NewReceipt(NewReceiptNotification), TriggerRAVRequest(RpcReplyPort), + GetUnaggregatedReceipts(RpcReplyPort), CloseAllocation, } @@ -67,6 +69,12 @@ impl Actor for SenderAllocation { _args: Self::Arguments, ) -> std::result::Result { let unaggregated_fees = self.calculate_unaggregated_fee().await?; + self.sender_account_ref + .cast(SenderAccountMessage::UpdateReceiptFees( + self.allocation_id, + unaggregated_fees.clone(), + ))?; + Ok(unaggregated_fees) } @@ -94,7 +102,7 @@ impl Actor for SenderAllocation { SenderAllocationMsg::NewReceipt(NewReceiptNotification { id, value: fees, .. }) => { - if id < state.last_id { + if id > state.last_id { state.last_id = id; state.value = state.value.checked_add(fees).unwrap_or_else(|| { // This should never happen, but if it does, we want to know about it. @@ -127,6 +135,12 @@ impl Actor for SenderAllocation { let _ = reply.send(state.clone()); } } + + SenderAllocationMsg::GetUnaggregatedReceipts(reply) => { + if !reply.is_closed() { + let _ = reply.send(state.clone()); + } + } SenderAllocationMsg::CloseAllocation => { self.rav_requester_single().await.inspect_err(|e| { error!( @@ -445,192 +459,192 @@ impl SenderAllocation { #[cfg(test)] mod tests { - - use std::collections::HashMap; - - use indexer_common::subgraph_client::DeploymentDetails; - use serde_json::json; - use tap_aggregator::server::run_server; - - use wiremock::{ - matchers::{body_string_contains, method}, - Mock, MockServer, ResponseTemplate, - }; - - use super::*; - use crate::tap::test_utils::{ - create_rav, create_received_receipt, store_rav, store_receipt, ALLOCATION_ID_0, INDEXER, - SENDER, SIGNER, TAP_EIP712_DOMAIN_SEPARATOR, - }; - - const DUMMY_URL: &str = "http://localhost:1234"; - - async fn create_sender_allocation( - pgpool: PgPool, - sender_aggregator_endpoint: String, - escrow_subgraph_endpoint: &str, - ) -> SenderAllocation { - let config = Box::leak(Box::new(config::Cli { - config: None, - ethereum: config::Ethereum { - indexer_address: INDEXER.1, - }, - tap: config::Tap { - rav_request_trigger_value: 100, - rav_request_timestamp_buffer_ms: 1, - rav_request_timeout_secs: 5, - ..Default::default() - }, - ..Default::default() - })); - - let escrow_subgraph = Box::leak(Box::new(SubgraphClient::new( - reqwest::Client::new(), - None, - DeploymentDetails::for_query_url(escrow_subgraph_endpoint).unwrap(), - ))); - - let escrow_accounts_eventual = Eventual::from_value(EscrowAccounts::new( - HashMap::from([(SENDER.1, 1000.into())]), - HashMap::from([(SENDER.1, vec![SIGNER.1])]), - )); - - let escrow_adapter = EscrowAdapter::new(escrow_accounts_eventual.clone(), SENDER.1); - - SenderAllocation::new( - config, - pgpool.clone(), - *ALLOCATION_ID_0, - SENDER.1, - escrow_accounts_eventual, - escrow_subgraph, - escrow_adapter, - TAP_EIP712_DOMAIN_SEPARATOR.clone(), - sender_aggregator_endpoint, - ) - .await - } - - /// Test that the sender_allocation correctly updates the unaggregated fees from the - /// database when there is no RAV in the database. - /// - /// The sender_allocation should consider all receipts found for the allocation and - /// sender. - #[sqlx::test(migrations = "../migrations")] - async fn test_update_unaggregated_fees_no_rav(pgpool: PgPool) { - let sender_allocation = - create_sender_allocation(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL).await; - - // Add receipts to the database. - for i in 1..10 { - let receipt = - create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i, i.into()).await; - store_receipt(&pgpool, receipt.signed_receipt()) - .await - .unwrap(); - } - - // Let the sender_allocation update the unaggregated fees from the database. - sender_allocation.update_unaggregated_fees().await.unwrap(); - - // Check that the unaggregated fees are correct. - assert_eq!( - sender_allocation.unaggregated_fees.lock().unwrap().value, - 45u128 - ); - } - - /// Test that the sender_allocation correctly updates the unaggregated fees from the - /// database when there is a RAV in the database as well as receipts which timestamp are lesser - /// and greater than the RAV's timestamp. - /// - /// The sender_allocation should only consider receipts with a timestamp greater - /// than the RAV's timestamp. - #[sqlx::test(migrations = "../migrations")] - async fn test_update_unaggregated_fees_with_rav(pgpool: PgPool) { - let sender_allocation = - create_sender_allocation(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL).await; - - // Add the RAV to the database. - // This RAV has timestamp 4. The sender_allocation should only consider receipts - // with a timestamp greater than 4. - let signed_rav = create_rav(*ALLOCATION_ID_0, SIGNER.0.clone(), 4, 10).await; - store_rav(&pgpool, signed_rav, SENDER.1).await.unwrap(); - - // Add receipts to the database. - for i in 1..10 { - let receipt = - create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i, i.into()).await; - store_receipt(&pgpool, receipt.signed_receipt()) - .await - .unwrap(); - } - - // Let the sender_allocation update the unaggregated fees from the database. - sender_allocation.update_unaggregated_fees().await.unwrap(); - - // Check that the unaggregated fees are correct. - assert_eq!( - sender_allocation.unaggregated_fees.lock().unwrap().value, - 35u128 - ); - } - - #[sqlx::test(migrations = "../migrations")] - async fn test_rav_requester_manual(pgpool: PgPool) { - // Start a TAP aggregator server. - let (handle, aggregator_endpoint) = run_server( - 0, - SIGNER.0.clone(), - vec![SIGNER.1].into_iter().collect(), - TAP_EIP712_DOMAIN_SEPARATOR.clone(), - 100 * 1024, - 100 * 1024, - 1, - ) - .await - .unwrap(); - - // Start a mock graphql server using wiremock - let mock_server = MockServer::start().await; - - // Mock result for TAP redeem txs for (allocation, sender) pair. - mock_server - .register( - Mock::given(method("POST")) - .and(body_string_contains("transactions")) - .respond_with( - ResponseTemplate::new(200) - .set_body_json(json!({ "data": { "transactions": []}})), - ), - ) - .await; - - // Create a sender_allocation. - let sender_allocation = create_sender_allocation( - pgpool.clone(), - "http://".to_owned() + &aggregator_endpoint.to_string(), - &mock_server.uri(), - ) - .await; - - // Add receipts to the database. - for i in 0..10 { - let receipt = - create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i + 1, i.into()).await; - store_receipt(&pgpool, receipt.signed_receipt()) - .await - .unwrap(); - } - - // Let the sender_allocation update the unaggregated fees from the database. - sender_allocation.update_unaggregated_fees().await.unwrap(); - - // Trigger a RAV request manually. - sender_allocation.rav_requester_single().await.unwrap(); - - // Stop the TAP aggregator server. - handle.stop().unwrap(); - handle.stopped().await; - } + // + // use std::collections::HashMap; + // + // use indexer_common::subgraph_client::DeploymentDetails; + // use serde_json::json; + // use tap_aggregator::server::run_server; + // + // use wiremock::{ + // matchers::{body_string_contains, method}, + // Mock, MockServer, ResponseTemplate, + // }; + // + // use super::*; + // use crate::tap::test_utils::{ + // create_rav, create_received_receipt, store_rav, store_receipt, ALLOCATION_ID_0, INDEXER, + // SENDER, SIGNER, TAP_EIP712_DOMAIN_SEPARATOR, + // }; + // + // const DUMMY_URL: &str = "http://localhost:1234"; + // + // async fn create_sender_allocation( + // pgpool: PgPool, + // sender_aggregator_endpoint: String, + // escrow_subgraph_endpoint: &str, + // ) -> SenderAllocation { + // let config = Box::leak(Box::new(config::Cli { + // config: None, + // ethereum: config::Ethereum { + // indexer_address: INDEXER.1, + // }, + // tap: config::Tap { + // rav_request_trigger_value: 100, + // rav_request_timestamp_buffer_ms: 1, + // rav_request_timeout_secs: 5, + // ..Default::default() + // }, + // ..Default::default() + // })); + // + // let escrow_subgraph = Box::leak(Box::new(SubgraphClient::new( + // reqwest::Client::new(), + // None, + // DeploymentDetails::for_query_url(escrow_subgraph_endpoint).unwrap(), + // ))); + // + // let escrow_accounts_eventual = Eventual::from_value(EscrowAccounts::new( + // HashMap::from([(SENDER.1, 1000.into())]), + // HashMap::from([(SENDER.1, vec![SIGNER.1])]), + // )); + // + // let escrow_adapter = EscrowAdapter::new(escrow_accounts_eventual.clone(), SENDER.1); + // + // SenderAllocation::new( + // config, + // pgpool.clone(), + // *ALLOCATION_ID_0, + // SENDER.1, + // escrow_accounts_eventual, + // escrow_subgraph, + // escrow_adapter, + // TAP_EIP712_DOMAIN_SEPARATOR.clone(), + // sender_aggregator_endpoint, + // ) + // .await + // } + // + // /// Test that the sender_allocation correctly updates the unaggregated fees from the + // /// database when there is no RAV in the database. + // /// + // /// The sender_allocation should consider all receipts found for the allocation and + // /// sender. + // #[sqlx::test(migrations = "../migrations")] + // async fn test_update_unaggregated_fees_no_rav(pgpool: PgPool) { + // let sender_allocation = + // create_sender_allocation(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL).await; + // + // // Add receipts to the database. + // for i in 1..10 { + // let receipt = + // create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i, i.into()).await; + // store_receipt(&pgpool, receipt.signed_receipt()) + // .await + // .unwrap(); + // } + // + // // Let the sender_allocation update the unaggregated fees from the database. + // sender_allocation.update_unaggregated_fees().await.unwrap(); + // + // // Check that the unaggregated fees are correct. + // assert_eq!( + // sender_allocation.unaggregated_fees.lock().unwrap().value, + // 45u128 + // ); + // } + // + // /// Test that the sender_allocation correctly updates the unaggregated fees from the + // /// database when there is a RAV in the database as well as receipts which timestamp are lesser + // /// and greater than the RAV's timestamp. + // /// + // /// The sender_allocation should only consider receipts with a timestamp greater + // /// than the RAV's timestamp. + // #[sqlx::test(migrations = "../migrations")] + // async fn test_update_unaggregated_fees_with_rav(pgpool: PgPool) { + // let sender_allocation = + // create_sender_allocation(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL).await; + // + // // Add the RAV to the database. + // // This RAV has timestamp 4. The sender_allocation should only consider receipts + // // with a timestamp greater than 4. + // let signed_rav = create_rav(*ALLOCATION_ID_0, SIGNER.0.clone(), 4, 10).await; + // store_rav(&pgpool, signed_rav, SENDER.1).await.unwrap(); + // + // // Add receipts to the database. + // for i in 1..10 { + // let receipt = + // create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i, i.into()).await; + // store_receipt(&pgpool, receipt.signed_receipt()) + // .await + // .unwrap(); + // } + // + // // Let the sender_allocation update the unaggregated fees from the database. + // sender_allocation.update_unaggregated_fees().await.unwrap(); + // + // // Check that the unaggregated fees are correct. + // assert_eq!( + // sender_allocation.unaggregated_fees.lock().unwrap().value, + // 35u128 + // ); + // } + // + // #[sqlx::test(migrations = "../migrations")] + // async fn test_rav_requester_manual(pgpool: PgPool) { + // // Start a TAP aggregator server. + // let (handle, aggregator_endpoint) = run_server( + // 0, + // SIGNER.0.clone(), + // vec![SIGNER.1].into_iter().collect(), + // TAP_EIP712_DOMAIN_SEPARATOR.clone(), + // 100 * 1024, + // 100 * 1024, + // 1, + // ) + // .await + // .unwrap(); + // + // // Start a mock graphql server using wiremock + // let mock_server = MockServer::start().await; + // + // // Mock result for TAP redeem txs for (allocation, sender) pair. + // mock_server + // .register( + // Mock::given(method("POST")) + // .and(body_string_contains("transactions")) + // .respond_with( + // ResponseTemplate::new(200) + // .set_body_json(json!({ "data": { "transactions": []}})), + // ), + // ) + // .await; + // + // // Create a sender_allocation. + // let sender_allocation = create_sender_allocation( + // pgpool.clone(), + // "http://".to_owned() + &aggregator_endpoint.to_string(), + // &mock_server.uri(), + // ) + // .await; + // + // // Add receipts to the database. + // for i in 0..10 { + // let receipt = + // create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i + 1, i.into()).await; + // store_receipt(&pgpool, receipt.signed_receipt()) + // .await + // .unwrap(); + // } + // + // // Let the sender_allocation update the unaggregated fees from the database. + // sender_allocation.update_unaggregated_fees().await.unwrap(); + // + // // Trigger a RAV request manually. + // sender_allocation.rav_requester_single().await.unwrap(); + // + // // Stop the TAP aggregator server. + // handle.stop().unwrap(); + // handle.stopped().await; + // } } From 4ae18e5b9a225d0f3774651ac17f1cd07db62477 Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Tue, 19 Mar 2024 22:18:43 -0300 Subject: [PATCH 11/41] test: wip tests on sender account working, need to allow parallel --- tap-agent/src/agent/sender_account.rs | 138 +++++++++-------------- tap-agent/src/agent/sender_allocation.rs | 1 - 2 files changed, 54 insertions(+), 85 deletions(-) diff --git a/tap-agent/src/agent/sender_account.rs b/tap-agent/src/agent/sender_account.rs index 228b3b5eb..7a5844d94 100644 --- a/tap-agent/src/agent/sender_account.rs +++ b/tap-agent/src/agent/sender_account.rs @@ -1,12 +1,11 @@ // Copyright 2023-, GraphOps and Semiotic Labs. // SPDX-License-Identifier: Apache-2.0 -use std::collections::{HashMap, HashSet}; +use std::collections::HashSet; use alloy_sol_types::Eip712Domain; use anyhow::Result; use eventuals::{Eventual, EventualExt, PipeHandle}; -use indexer_common::allocations::Allocation; use indexer_common::{escrow_accounts::EscrowAccounts, prelude::SubgraphClient}; use ractor::{call, Actor, ActorProcessingErr, ActorRef, RpcReplyPort}; use sqlx::PgPool; @@ -220,17 +219,13 @@ mod tests { use crate::agent::sender_accounts_manager::NewReceiptNotification; use alloy_primitives::hex::ToHex; use bigdecimal::{num_bigint::ToBigInt, ToPrimitive}; - use ethereum_types::U256; - use indexer_common::allocations::{AllocationStatus, SubgraphDeployment}; use indexer_common::subgraph_client::DeploymentDetails; use serde_json::json; use std::collections::HashMap; use std::str::FromStr; - use std::sync::Arc; use std::time::Duration; use tap_aggregator::server::run_server; use tap_core::{rav::ReceiptAggregateVoucher, signed_message::EIP712SignedMessage}; - use thegraph::types::DeploymentId; use wiremock::{ matchers::{body_string_contains, method}, Mock, MockServer, ResponseTemplate, @@ -279,9 +274,6 @@ mod tests { HashMap::from([(SENDER.1, vec![SIGNER.1])]), )); - let allocation_id = - Address::from_str("0xdd975e30aafebb143e54d215db8a3e8fd916a701").unwrap(); - let indexer_allocations = Eventual::from_value(HashSet::from([ *ALLOCATION_ID_0, *ALLOCATION_ID_1, @@ -299,10 +291,9 @@ mod tests { sender_aggregator_endpoint, ); - let (sender, _handle) = - SenderAccount::spawn(Some(SENDER.1.to_string()), sender, HashSet::new()) - .await - .unwrap(); + let (sender, _handle) = SenderAccount::spawn(None, sender, HashSet::new()) + .await + .unwrap(); // await for the allocations to be created ractor::concurrency::sleep(Duration::from_millis(100)).await; @@ -509,7 +500,7 @@ mod tests { allocation_id: *ALLOCATION_ID_0, signer_address: SIGNER.1, id: i, - timestamp_ns: i + 1 , + timestamp_ns: i + 1, value, }; allocation @@ -679,74 +670,53 @@ mod tests { handle.stopped().await; } - // #[sqlx::test(migrations = "../migrations")] - // async fn test_sender_unaggregated_fees(pgpool: PgPool) { - // // Create a sender_account. - // let sender_account = Arc::new( - // create_sender_with_allocations(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL).await, - // ); - // - // // Closure that adds a number of receipts to an allocation. - // let add_receipts = |allocation_id: Address, iterations: u64| { - // let sender_account = sender_account.clone(); - // - // async move { - // let mut total_value = 0; - // for i in 0..iterations { - // let value = (i + 10) as u128; - // - // let id = sender_account.unaggregated_fees.lock().unwrap().last_id + 1; - // - // sender_account - // .handle_new_receipt_notification(NewReceiptNotification { - // allocation_id, - // signer_address: SIGNER.1, - // id, - // timestamp_ns: i + 1, - // value, - // }) - // .await; - // - // total_value += value; - // } - // - // assert_eq!( - // sender_account - // .allocations - // .lock() - // .unwrap() - // .get(&allocation_id) - // .unwrap() - // .as_active() - // .unwrap() - // .get_unaggregated_fees() - // .value, - // total_value - // ); - // - // total_value - // } - // }; - // - // // Add receipts to the database for allocation_0 - // let total_value_0 = add_receipts(*ALLOCATION_ID_0, 9).await; - // - // // Add receipts to the database for allocation_1 - // let total_value_1 = add_receipts(*ALLOCATION_ID_1, 10).await; - // - // // Add receipts to the database for allocation_2 - // let total_value_2 = add_receipts(*ALLOCATION_ID_2, 8).await; - // - // // Get the heaviest allocation. - // let heaviest_allocation = sender_account.get_heaviest_allocation().unwrap(); - // - // // Check that the heaviest allocation is correct. - // assert_eq!(heaviest_allocation.get_allocation_id(), *ALLOCATION_ID_1); - // - // // Check that the sender's unaggregated fees value is correct. - // assert_eq!( - // sender_account.unaggregated_fees.lock().unwrap().value, - // total_value_0 + total_value_1 + total_value_2 - // ); - // } + #[sqlx::test(migrations = "../migrations")] + async fn test_sender_unaggregated_fees(pgpool: PgPool) { + // Create a sender_account. + let sender_account = + create_sender_with_allocations(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL).await; + + // Closure that adds a number of receipts to an allocation. + let update_receipt_fees = |allocation_id: Address, value: u128| { + let sender_account = sender_account.clone(); + sender_account + .cast(SenderAccountMessage::UpdateReceiptFees( + allocation_id, + UnaggregatedReceipts { + value, + ..Default::default() + }, + )) + .unwrap(); + + value + }; + + // Add receipts to the database for allocation_0 + let total_value_0 = update_receipt_fees(*ALLOCATION_ID_0, 90); + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + + // Add receipts to the database for allocation_1 + let total_value_1 = update_receipt_fees(*ALLOCATION_ID_1, 100); + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + + // Add receipts to the database for allocation_2 + let total_value_2 = update_receipt_fees(*ALLOCATION_ID_2, 80); + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + + let allocation_tracker = + call!(sender_account, SenderAccountMessage::GetAllocationTracker).unwrap(); + + // Get the heaviest allocation. + let heaviest_allocation = allocation_tracker.get_heaviest_allocation_id().unwrap(); + + // Check that the heaviest allocation is correct. + assert_eq!(heaviest_allocation, *ALLOCATION_ID_1); + + // Check that the sender's unaggregated fees value is correct. + assert_eq!( + allocation_tracker.get_total_fee(), + total_value_0 + total_value_1 + total_value_2 + ); + } } diff --git a/tap-agent/src/agent/sender_allocation.rs b/tap-agent/src/agent/sender_allocation.rs index 3828505d8..332881380 100644 --- a/tap-agent/src/agent/sender_allocation.rs +++ b/tap-agent/src/agent/sender_allocation.rs @@ -24,7 +24,6 @@ use tap_core::{ use thegraph::types::Address; use tracing::{error, warn}; -use crate::agent::allocation_id_tracker::AllocationIdTracker; use crate::agent::sender_account::SenderAccountMessage; use crate::agent::sender_accounts_manager::NewReceiptNotification; use crate::agent::unaggregated_receipts::UnaggregatedReceipts; From 4b986d1d9dd0f4a7e9dd48ab8f8b0b394364e016 Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Wed, 20 Mar 2024 12:16:20 -0300 Subject: [PATCH 12/41] refactor: use sender accounts manager args --- tap-agent/src/agent.rs | 93 +++--- tap-agent/src/agent/sender_account.rs | 4 +- .../src/agent/sender_accounts_manager.rs | 264 +++++++++--------- tap-agent/src/main.rs | 37 ++- tap-agent/tests/sender_allocation_tests.rs | 1 + 5 files changed, 221 insertions(+), 178 deletions(-) create mode 100644 tap-agent/tests/sender_allocation_tests.rs diff --git a/tap-agent/src/agent.rs b/tap-agent/src/agent.rs index b51e74c9b..ff4f8c15f 100644 --- a/tap-agent/src/agent.rs +++ b/tap-agent/src/agent.rs @@ -3,12 +3,16 @@ use std::time::Duration; -use alloy_sol_types::eip712_domain; use indexer_common::prelude::{ escrow_accounts, indexer_allocations, DeploymentDetails, SubgraphClient, }; +use ractor::{Actor, ActorRef}; -use crate::{aggregator_endpoints, config, database}; +use crate::agent::sender_accounts_manager::{ + SenderAccountsManagerArgs, SenderAccountsManagerMessage, +}; +use crate::config::{Cli, EscrowSubgraph, Ethereum, IndexerInfrastructure, NetworkSubgraph, Tap}; +use crate::{aggregator_endpoints, database, CONFIG, EIP_712_DOMAIN}; use sender_accounts_manager::SenderAccountsManager; mod allocation_id_tracker; @@ -17,81 +21,100 @@ pub mod sender_accounts_manager; mod sender_allocation; mod unaggregated_receipts; -pub async fn start_agent(config: &'static config::Cli) -> SenderAccountsManager { - let pgpool = database::connect(&config.postgres).await; +pub async fn start_agent() -> ActorRef { + let Cli { + ethereum: Ethereum { indexer_address }, + indexer_infrastructure: + IndexerInfrastructure { + graph_node_query_endpoint, + graph_node_status_endpoint, + .. + }, + postgres, + network_subgraph: + NetworkSubgraph { + network_subgraph_deployment, + network_subgraph_endpoint, + allocation_syncing_interval_ms, + }, + escrow_subgraph: + EscrowSubgraph { + escrow_subgraph_deployment, + escrow_subgraph_endpoint, + escrow_syncing_interval_ms, + }, + tap: Tap { + sender_aggregator_endpoints_file, + .. + }, + .. + } = &*CONFIG; + let pgpool = database::connect(postgres).await; let http_client = reqwest::Client::new(); let network_subgraph = Box::leak(Box::new(SubgraphClient::new( http_client.clone(), - config - .network_subgraph - .network_subgraph_deployment + network_subgraph_deployment .map(|deployment| { DeploymentDetails::for_graph_node( - &config.indexer_infrastructure.graph_node_status_endpoint, - &config.indexer_infrastructure.graph_node_query_endpoint, + graph_node_status_endpoint, + graph_node_query_endpoint, deployment, ) }) .transpose() .expect("Failed to parse graph node query endpoint and network subgraph deployment"), - DeploymentDetails::for_query_url(&config.network_subgraph.network_subgraph_endpoint) + DeploymentDetails::for_query_url(network_subgraph_endpoint) .expect("Failed to parse network subgraph endpoint"), ))); let indexer_allocations = indexer_allocations( network_subgraph, - config.ethereum.indexer_address, + *indexer_address, 1, - Duration::from_millis(config.network_subgraph.allocation_syncing_interval_ms), + Duration::from_millis(*allocation_syncing_interval_ms), ); let escrow_subgraph = Box::leak(Box::new(SubgraphClient::new( http_client.clone(), - config - .escrow_subgraph - .escrow_subgraph_deployment + escrow_subgraph_deployment .map(|deployment| { DeploymentDetails::for_graph_node( - &config.indexer_infrastructure.graph_node_status_endpoint, - &config.indexer_infrastructure.graph_node_query_endpoint, + graph_node_status_endpoint, + graph_node_query_endpoint, deployment, ) }) .transpose() .expect("Failed to parse graph node query endpoint and escrow subgraph deployment"), - DeploymentDetails::for_query_url(&config.escrow_subgraph.escrow_subgraph_endpoint) + DeploymentDetails::for_query_url(escrow_subgraph_endpoint) .expect("Failed to parse escrow subgraph endpoint"), ))); let escrow_accounts = escrow_accounts( escrow_subgraph, - config.ethereum.indexer_address, - Duration::from_millis(config.escrow_subgraph.escrow_syncing_interval_ms), + *indexer_address, + Duration::from_millis(*escrow_syncing_interval_ms), false, ); // TODO: replace with a proper implementation once the gateway registry contract is ready - let sender_aggregator_endpoints = aggregator_endpoints::load_aggregator_endpoints( - config.tap.sender_aggregator_endpoints_file.clone(), - ); - - let tap_eip712_domain_separator = eip712_domain! { - name: "TAP", - version: "1", - chain_id: config.receipts.receipts_verifier_chain_id, - verifying_contract: config.receipts.receipts_verifier_address, - }; + let sender_aggregator_endpoints = + aggregator_endpoints::load_aggregator_endpoints(sender_aggregator_endpoints_file.clone()); - SenderAccountsManager::new( - config, + let args = SenderAccountsManagerArgs { + config: &CONFIG, + domain_separator: EIP_712_DOMAIN.clone(), pgpool, indexer_allocations, escrow_accounts, escrow_subgraph, - tap_eip712_domain_separator, sender_aggregator_endpoints, - ) - .await + }; + + let (manager, _) = SenderAccountsManager::spawn(None, SenderAccountsManager, args) + .await + .expect("Failed to start sender accounts manager actor."); + manager } diff --git a/tap-agent/src/agent/sender_account.rs b/tap-agent/src/agent/sender_account.rs index 7a5844d94..9b9a81e75 100644 --- a/tap-agent/src/agent/sender_account.rs +++ b/tap-agent/src/agent/sender_account.rs @@ -324,7 +324,7 @@ mod tests { let allocation = ActorRef::::where_is(format!( "{sender}:{allocation_id}", sender = SENDER.1, - allocation_id = ALLOCATION_ID_0.to_string() + allocation_id = *ALLOCATION_ID_0 )) .unwrap(); @@ -478,7 +478,7 @@ mod tests { let allocation = ActorRef::::where_is(format!( "{sender}:{allocation_id}", sender = SENDER.1, - allocation_id = ALLOCATION_ID_0.to_string() + allocation_id = *ALLOCATION_ID_0 )) .unwrap(); diff --git a/tap-agent/src/agent/sender_accounts_manager.rs b/tap-agent/src/agent/sender_accounts_manager.rs index 733268156..d6f745367 100644 --- a/tap-agent/src/agent/sender_accounts_manager.rs +++ b/tap-agent/src/agent/sender_accounts_manager.rs @@ -29,41 +29,61 @@ pub struct NewReceiptNotification { pub value: u128, } -pub struct SenderAccountsManager { - config: &'static config::Cli, - pgpool: PgPool, - indexer_allocations: Eventual>, - escrow_accounts: Eventual, - escrow_subgraph: &'static SubgraphClient, - tap_eip712_domain_separator: Eip712Domain, - sender_aggregator_endpoints: HashMap, -} +pub struct SenderAccountsManager; -pub enum NetworkMessage { +pub enum SenderAccountsManagerMessage { UpdateSenderAccounts(HashSet
), CreateSenderAccount(Address, HashSet
), } +pub struct SenderAccountsManagerArgs { + pub config: &'static config::Cli, + pub domain_separator: Eip712Domain, + + pub pgpool: PgPool, + pub indexer_allocations: Eventual>, + pub escrow_accounts: Eventual, + pub escrow_subgraph: &'static SubgraphClient, + pub sender_aggregator_endpoints: HashMap, +} + pub struct State { sender_ids: HashSet
, new_receipts_watcher_handle: tokio::task::JoinHandle<()>, _eligible_allocations_senders_pipe: PipeHandle, + + config: &'static config::Cli, + domain_separator: Eip712Domain, + pgpool: PgPool, + indexer_allocations: Eventual>, + escrow_accounts: Eventual, + escrow_subgraph: &'static SubgraphClient, + sender_aggregator_endpoints: HashMap, } #[async_trait::async_trait] impl Actor for SenderAccountsManager { - type Msg = NetworkMessage; + type Msg = SenderAccountsManagerMessage; type State = State; - type Arguments = (); + type Arguments = SenderAccountsManagerArgs; async fn pre_start( &self, myself: ActorRef, - _: Self::Arguments, + SenderAccountsManagerArgs { + config, + domain_separator, + indexer_allocations, + pgpool, + escrow_accounts, + escrow_subgraph, + sender_aggregator_endpoints, + }: Self::Arguments, ) -> std::result::Result { - let mut pglistener = PgListener::connect_with(&self.pgpool.clone()) - .await - .unwrap(); + let indexer_allocations = indexer_allocations.map(|allocations| async move { + allocations.keys().cloned().collect::>() + }); + let mut pglistener = PgListener::connect_with(&pgpool.clone()).await.unwrap(); pglistener .listen("scalar_tap_receipt_notification") .await @@ -72,40 +92,45 @@ impl Actor for SenderAccountsManager { 'scalar_tap_receipt_notification'", ); // Start the new_receipts_watcher task that will consume from the `pglistener` - let new_receipts_watcher_handle = tokio::spawn(Self::new_receipts_watcher( - pglistener, - self.escrow_accounts.clone(), - )); + let new_receipts_watcher_handle = + tokio::spawn(new_receipts_watcher(pglistener, escrow_accounts.clone())); let clone = myself.clone(); let _eligible_allocations_senders_pipe = - self.escrow_accounts - .clone() - .pipe_async(move |escrow_accounts| { - let myself = clone.clone(); - - async move { - myself - .cast(NetworkMessage::UpdateSenderAccounts( - escrow_accounts.get_senders(), - )) - .unwrap_or_else(|e| { - error!("Error while updating sender_accounts: {:?}", e); - }); - } - }); - let sender_allocation = self.get_pending_sender_allocation_id().await; + escrow_accounts.clone().pipe_async(move |escrow_accounts| { + let myself = clone.clone(); + + async move { + myself + .cast(SenderAccountsManagerMessage::UpdateSenderAccounts( + escrow_accounts.get_senders(), + )) + .unwrap_or_else(|e| { + error!("Error while updating sender_accounts: {:?}", e); + }); + } + }); + + let state = State { + config, + domain_separator, + sender_ids: HashSet::new(), + new_receipts_watcher_handle, + _eligible_allocations_senders_pipe, + pgpool, + indexer_allocations, + escrow_accounts, + escrow_subgraph, + sender_aggregator_endpoints, + }; + let sender_allocation = state.get_pending_sender_allocation_id().await; for (sender_id, allocation_ids) in sender_allocation { - myself.cast(NetworkMessage::CreateSenderAccount( + myself.cast(SenderAccountsManagerMessage::CreateSenderAccount( sender_id, allocation_ids, ))?; } - Ok(State { - sender_ids: HashSet::new(), - new_receipts_watcher_handle, - _eligible_allocations_senders_pipe, - }) + Ok(state) } async fn post_stop( &self, @@ -125,10 +150,13 @@ impl Actor for SenderAccountsManager { state: &mut Self::State, ) -> std::result::Result<(), ActorProcessingErr> { match msg { - NetworkMessage::UpdateSenderAccounts(target_senders) => { + SenderAccountsManagerMessage::UpdateSenderAccounts(target_senders) => { // Create new sender accounts for sender in target_senders.difference(&state.sender_ids) { - myself.cast(NetworkMessage::CreateSenderAccount(*sender, HashSet::new()))?; + myself.cast(SenderAccountsManagerMessage::CreateSenderAccount( + *sender, + HashSet::new(), + ))?; } // Remove sender accounts @@ -142,8 +170,8 @@ impl Actor for SenderAccountsManager { state.sender_ids = target_senders; } - NetworkMessage::CreateSenderAccount(sender_id, allocation_ids) => { - let sender_account = self.new_sender_account(&sender_id)?; + SenderAccountsManagerMessage::CreateSenderAccount(sender_id, allocation_ids) => { + let sender_account = state.new_sender_account(&sender_id)?; SenderAccount::spawn_linked( Some(sender_id.to_string()), sender_account, @@ -157,30 +185,7 @@ impl Actor for SenderAccountsManager { } } -impl SenderAccountsManager { - pub async fn new( - config: &'static config::Cli, - pgpool: PgPool, - indexer_allocations: Eventual>, - escrow_accounts: Eventual, - escrow_subgraph: &'static SubgraphClient, - tap_eip712_domain_separator: Eip712Domain, - sender_aggregator_endpoints: HashMap, - ) -> Self { - let indexer_allocations = indexer_allocations.map(|allocations| async move { - allocations.keys().cloned().collect::>() - }); - Self { - config, - pgpool, - indexer_allocations, - escrow_accounts, - escrow_subgraph, - tap_eip712_domain_separator, - sender_aggregator_endpoints, - } - } - +impl State { async fn get_pending_sender_allocation_id(&self) -> HashMap> { let escrow_accounts_snapshot = self .escrow_accounts @@ -287,7 +292,7 @@ impl SenderAccountsManager { self.escrow_accounts.clone(), self.indexer_allocations.clone(), self.escrow_subgraph, - self.tap_eip712_domain_separator.clone(), + self.domain_separator.clone(), self.sender_aggregator_endpoints .get(sender_id) .ok_or_else(|| { @@ -299,60 +304,63 @@ impl SenderAccountsManager { .clone(), )) } +} - /// Continuously listens for new receipt notifications from Postgres and forwards them to the - /// corresponding SenderAccount. - async fn new_receipts_watcher( - mut pglistener: PgListener, - escrow_accounts: Eventual, - ) { - loop { - // TODO: recover from errors or shutdown the whole program? - let pg_notification = pglistener.recv().await.expect( - "should be able to receive Postgres Notify events on the channel \ +/// Continuously listens for new receipt notifications from Postgres and forwards them to the +/// corresponding SenderAccount. +async fn new_receipts_watcher( + mut pglistener: PgListener, + escrow_accounts: Eventual, +) { + loop { + // TODO: recover from errors or shutdown the whole program? + let pg_notification = pglistener.recv().await.expect( + "should be able to receive Postgres Notify events on the channel \ 'scalar_tap_receipt_notification'", - ); - let new_receipt_notification: NewReceiptNotification = - serde_json::from_str(pg_notification.payload()).expect( - "should be able to deserialize the Postgres Notify event payload as a \ + ); + let new_receipt_notification: NewReceiptNotification = + serde_json::from_str(pg_notification.payload()).expect( + "should be able to deserialize the Postgres Notify event payload as a \ NewReceiptNotification", - ); + ); - let sender_address = escrow_accounts - .value() - .await - .expect("should be able to get escrow accounts") - .get_sender_for_signer(&new_receipt_notification.signer_address); - - let sender_address = match sender_address { - Ok(sender_address) => sender_address, - Err(_) => { - error!( - "No sender address found for receipt signer address {}. \ + let sender_address = escrow_accounts + .value() + .await + .expect("should be able to get escrow accounts") + .get_sender_for_signer(&new_receipt_notification.signer_address); + + let sender_address = match sender_address { + Ok(sender_address) => sender_address, + Err(_) => { + error!( + "No sender address found for receipt signer address {}. \ This should not happen.", - new_receipt_notification.signer_address - ); - // TODO: save the receipt in the failed receipts table? - continue; - } - }; - let allocation_id = &new_receipt_notification.allocation_id; - - if let Some(sender_allocation) = ActorRef::::where_is(format!( - "{sender_address}:{allocation_id}" - )) { - if let Err(e) = sender_allocation - .cast(SenderAllocationMsg::NewReceipt(new_receipt_notification)) - { - error!("Error while forwarding new receipt notification to sender_allocation: {:?}", e); - } - } else { - warn!( - "No sender_allocation_manager found for sender_address {} to process new \ - receipt notification. This should not happen.", - sender_address + new_receipt_notification.signer_address ); + // TODO: save the receipt in the failed receipts table? + continue; } + }; + let allocation_id = &new_receipt_notification.allocation_id; + + if let Some(sender_allocation) = + ActorRef::::where_is(format!("{sender_address}:{allocation_id}")) + { + if let Err(e) = + sender_allocation.cast(SenderAllocationMsg::NewReceipt(new_receipt_notification)) + { + error!( + "Error while forwarding new receipt notification to sender_allocation: {:?}", + e + ); + } + } else { + warn!( + "No sender_allocation_manager found for sender_address {} to process new \ + receipt notification. This should not happen.", + sender_address + ); } } } @@ -420,17 +428,19 @@ mod tests { DeploymentDetails::for_query_url(&mock_server.uri()).unwrap(), ))); - let sender_account = SenderAccountsManager::new( + let args = SenderAccountsManagerArgs { config, - pgpool.clone(), - indexer_allocations_eventual, - escrow_accounts_eventual, + domain_separator: TAP_EIP712_DOMAIN_SEPARATOR.clone(), + pgpool: pgpool.clone(), + indexer_allocations: indexer_allocations_eventual, + escrow_accounts: escrow_accounts_eventual, escrow_subgraph, - TAP_EIP712_DOMAIN_SEPARATOR.clone(), - HashMap::from([(SENDER.1, String::from("http://localhost:8000"))]), - ) - .await; - SenderAccountsManager::spawn(None, sender_account, ()) + sender_aggregator_endpoints: HashMap::from([( + SENDER.1, + String::from("http://localhost:8000"), + )]), + }; + SenderAccountsManager::spawn(None, SenderAccountsManager, args) .await .unwrap(); diff --git a/tap-agent/src/main.rs b/tap-agent/src/main.rs index a99ac21df..4d5b2c59f 100644 --- a/tap-agent/src/main.rs +++ b/tap-agent/src/main.rs @@ -1,6 +1,7 @@ // Copyright 2023-, GraphOps and Semiotic Labs. // SPDX-License-Identifier: Apache-2.0 +use alloy_sol_types::{eip712_domain, Eip712Domain}; use anyhow::Result; use lazy_static::lazy_static; use tokio::signal::unix::{signal, SignalKind}; @@ -16,6 +17,12 @@ mod tap; lazy_static! { pub static ref CONFIG: Cli = Cli::args(); + pub static ref EIP_712_DOMAIN: Eip712Domain = eip712_domain! { + name: "TAP", + version: "1", + chain_id: CONFIG.receipts.receipts_verifier_chain_id, + verifying_contract: CONFIG.receipts.receipts_verifier_address, + }; } #[tokio::main] @@ -24,22 +31,24 @@ async fn main() -> Result<()> { lazy_static::initialize(&CONFIG); debug!("Config: {:?}", *CONFIG); - { - let _manager = agent::start_agent(&CONFIG).await; - info!("TAP Agent started."); + let manager = agent::start_agent().await; + info!("TAP Agent started."); - // Have tokio wait for SIGTERM or SIGINT. - let mut signal_sigint = signal(SignalKind::interrupt())?; - let mut signal_sigterm = signal(SignalKind::terminate())?; - tokio::select! { - _ = signal_sigint.recv() => debug!("Received SIGINT."), - _ = signal_sigterm.recv() => debug!("Received SIGTERM."), - } - - // If we're here, we've received a signal to exit. - info!("Shutting down..."); + // Have tokio wait for SIGTERM or SIGINT. + let mut signal_sigint = signal(SignalKind::interrupt())?; + let mut signal_sigterm = signal(SignalKind::terminate())?; + tokio::select! { + _ = signal_sigint.recv() => debug!("Received SIGINT."), + _ = signal_sigterm.recv() => debug!("Received SIGTERM."), } - // Manager should be successfully dropped here. + // If we're here, we've received a signal to exit. + info!("Shutting down..."); + + // We don't want our actor to run any shutdown logic, so we kill it. + manager + .kill_and_wait(None) + .await + .expect("Failed to kill manager."); // Stop the server and wait for it to finish gracefully. debug!("Goodbye!"); diff --git a/tap-agent/tests/sender_allocation_tests.rs b/tap-agent/tests/sender_allocation_tests.rs new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/tap-agent/tests/sender_allocation_tests.rs @@ -0,0 +1 @@ + From d32b552ff4b2fea3c9c02239bd912a40983e21a4 Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Mon, 1 Apr 2024 19:34:18 -0300 Subject: [PATCH 13/41] test: finish integration tests Signed-off-by: Gustavo Inacio --- tap-agent/src/agent/sender_account.rs | 189 +++++--- .../src/agent/sender_accounts_manager.rs | 18 +- tap-agent/src/agent/sender_allocation.rs | 439 ++++++++++-------- 3 files changed, 369 insertions(+), 277 deletions(-) diff --git a/tap-agent/src/agent/sender_account.rs b/tap-agent/src/agent/sender_account.rs index 9b9a81e75..c3b61f33d 100644 --- a/tap-agent/src/agent/sender_account.rs +++ b/tap-agent/src/agent/sender_account.rs @@ -7,14 +7,14 @@ use alloy_sol_types::Eip712Domain; use anyhow::Result; use eventuals::{Eventual, EventualExt, PipeHandle}; use indexer_common::{escrow_accounts::EscrowAccounts, prelude::SubgraphClient}; -use ractor::{call, Actor, ActorProcessingErr, ActorRef, RpcReplyPort}; +use ractor::{call, Actor, ActorProcessingErr, ActorRef}; use sqlx::PgPool; use thegraph::types::Address; use tracing::error; use super::sender_allocation::SenderAllocation; use crate::agent::allocation_id_tracker::AllocationIdTracker; -use crate::agent::sender_allocation::SenderAllocationMsg; +use crate::agent::sender_allocation::SenderAllocationMessage; use crate::agent::unaggregated_receipts::UnaggregatedReceipts; use crate::{ config::{self}, @@ -26,7 +26,8 @@ pub enum SenderAccountMessage { UpdateAllocationIds(HashSet
), RemoveSenderAccount, UpdateReceiptFees(Address, UnaggregatedReceipts), - GetAllocationTracker(RpcReplyPort), + #[cfg(test)] + GetAllocationTracker(ractor::RpcReplyPort), } /// A SenderAccount manages the receipts accounting between the indexer and the sender across @@ -51,23 +52,54 @@ pub struct SenderAccount { sender: Address, sender_aggregator_endpoint: String, } - pub struct State { + prefix: Option, allocation_id_tracker: AllocationIdTracker, allocation_ids: HashSet
, _indexer_allocations_handle: PipeHandle, + sender: Address, +} + +impl State { + fn format_sender_allocation(&self, allocation_id: &Address) -> String { + let mut sender_allocation_id = String::new(); + if let Some(prefix) = &self.prefix { + sender_allocation_id.push_str(prefix); + sender_allocation_id.push(':'); + } + sender_allocation_id.push_str(&format!("{}:{}", self.sender, allocation_id)); + sender_allocation_id + } + + async fn rav_requester_single(&mut self) -> Result<()> { + let Some(allocation_id) = self.allocation_id_tracker.get_heaviest_allocation_id() else { + anyhow::bail!("Error while getting allocation with most unaggregated fees"); + }; + let sender_allocation_id = self.format_sender_allocation(&allocation_id); + let allocation = ActorRef::::where_is(sender_allocation_id); + + let Some(allocation) = allocation else { + anyhow::bail!("Error while getting allocation with most unaggregated fees"); + }; + // we call and wait for the response so we don't process anymore update + let result = call!(allocation, SenderAllocationMessage::TriggerRAVRequest)?; + + self.allocation_id_tracker + .add_or_update(allocation_id, result.value); + Ok(()) + } } #[async_trait::async_trait] impl Actor for SenderAccount { type Msg = SenderAccountMessage; type State = State; - type Arguments = HashSet
; + type Arguments = (HashSet
, Option); async fn pre_start( &self, myself: ActorRef, - allocation_ids: Self::Arguments, + (allocation_ids, prefix): Self::Arguments, ) -> std::result::Result { let clone = myself.clone(); let _indexer_allocations_handle = @@ -94,6 +126,8 @@ impl Actor for SenderAccount { allocation_id_tracker: AllocationIdTracker::new(), allocation_ids, _indexer_allocations_handle, + prefix, + sender: self.sender, }) } @@ -110,7 +144,7 @@ impl Actor for SenderAccount { tracker.add_or_update(allocation_id, unaggregated_fees.value); if tracker.get_total_fee() >= self.config.tap.rav_request_trigger_value.into() { - self.rav_requester_single(tracker).await?; + state.rav_requester_single().await?; } } SenderAccountMessage::CreateSenderAllocation(allocation_id) => { @@ -127,10 +161,9 @@ impl Actor for SenderAccount { myself.clone(), ) .await; - let sender_id = self.sender; - let (_actor, _handle) = SenderAllocation::spawn_linked( - Some(format!("{sender_id}:{allocation_id}")), + SenderAllocation::spawn_linked( + Some(state.format_sender_allocation(&allocation_id)), sender_allocation, (), myself.get_cell(), @@ -144,17 +177,17 @@ impl Actor for SenderAccount { } // Remove sender allocations - let sender = self.sender; for allocation_id in state.allocation_ids.difference(&allocation_ids) { - if let Some(sender_handle) = ActorRef::::where_is(format!( - "{sender}:{allocation_id}" - )) { - sender_handle.cast(SenderAllocationMsg::CloseAllocation)?; + if let Some(sender_handle) = ActorRef::::where_is( + state.format_sender_allocation(allocation_id), + ) { + sender_handle.cast(SenderAllocationMessage::CloseAllocation)?; } } state.allocation_ids = allocation_ids; } + #[cfg(test)] SenderAccountMessage::GetAllocationTracker(reply) => { if !reply.is_closed() { let _ = reply.send(state.allocation_id_tracker.clone()); @@ -191,27 +224,6 @@ impl SenderAccount { sender: sender_id, } } - - async fn rav_requester_single( - &self, - heaviest_allocation: &mut AllocationIdTracker, - ) -> Result<()> { - let Some(allocation_id) = heaviest_allocation.get_heaviest_allocation_id() else { - anyhow::bail!("Error while getting allocation with most unaggregated fees"); - }; - let sender_id = self.sender; - let allocation = - ActorRef::::where_is(format!("{sender_id}:{allocation_id}")); - - let Some(allocation) = allocation else { - anyhow::bail!("Error while getting allocation with most unaggregated fees"); - }; - // we call and wait for the response so we don't process anymore update - let result = call!(allocation, SenderAllocationMsg::TriggerRAVRequest)?; - - heaviest_allocation.add_or_update(allocation_id, result.value); - Ok(()) - } } #[cfg(test)] @@ -220,12 +232,14 @@ mod tests { use alloy_primitives::hex::ToHex; use bigdecimal::{num_bigint::ToBigInt, ToPrimitive}; use indexer_common::subgraph_client::DeploymentDetails; + use ractor::cast; use serde_json::json; - use std::collections::HashMap; use std::str::FromStr; use std::time::Duration; + use std::{collections::HashMap, sync::atomic::AtomicU32}; use tap_aggregator::server::run_server; use tap_core::{rav::ReceiptAggregateVoucher, signed_message::EIP712SignedMessage}; + use tokio::task::JoinHandle; use wiremock::{ matchers::{body_string_contains, method}, Mock, MockServer, ResponseTemplate, @@ -241,6 +255,7 @@ mod tests { const DUMMY_URL: &str = "http://localhost:1234"; const VALUE_PER_RECEIPT: u64 = 100; const TRIGGER_VALUE: u64 = 500; + static PREFIX_ID: AtomicU32 = AtomicU32::new(0); // To help with testing from other modules. @@ -248,7 +263,7 @@ mod tests { pgpool: PgPool, sender_aggregator_endpoint: String, escrow_subgraph_endpoint: &str, - ) -> ActorRef { + ) -> (ActorRef, JoinHandle<()>, String) { let config = Box::leak(Box::new(config::Cli { config: None, ethereum: config::Ethereum { @@ -291,14 +306,23 @@ mod tests { sender_aggregator_endpoint, ); - let (sender, _handle) = SenderAccount::spawn(None, sender, HashSet::new()) - .await - .unwrap(); + let prefix = format!( + "test-{}", + PREFIX_ID.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + ); + + let (sender, handle) = SenderAccount::spawn( + Some(prefix.clone()), + sender, + (HashSet::new(), Some(prefix.clone())), + ) + .await + .unwrap(); // await for the allocations to be created ractor::concurrency::sleep(Duration::from_millis(100)).await; - sender + (sender, handle, prefix) } /// Test that the sender_account correctly ignores new receipt notifications with @@ -318,11 +342,11 @@ mod tests { expected_unaggregated_fees += u128::from(i); } - let sender = + let (sender, handle, prefix) = create_sender_with_allocations(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL).await; - let allocation = ActorRef::::where_is(format!( - "{sender}:{allocation_id}", + let allocation = ActorRef::::where_is(format!( + "{prefix}:{sender}:{allocation_id}", sender = SENDER.1, allocation_id = *ALLOCATION_ID_0 )) @@ -336,7 +360,7 @@ mod tests { ); let allocation_unaggregated_fees = - call!(allocation, SenderAllocationMsg::GetUnaggregatedReceipts).unwrap(); + call!(allocation, SenderAllocationMessage::GetUnaggregatedReceipts).unwrap(); // Check that the allocation's unaggregated fees are correct. assert_eq!( @@ -355,12 +379,14 @@ mod tests { value: 19, }; allocation - .cast(SenderAllocationMsg::NewReceipt(new_receipt_notification)) + .cast(SenderAllocationMessage::NewReceipt( + new_receipt_notification, + )) .unwrap(); ractor::concurrency::sleep(Duration::from_millis(10)).await; let allocation_unaggregated_fees = - call!(allocation, SenderAllocationMsg::GetUnaggregatedReceipts).unwrap(); + call!(allocation, SenderAllocationMessage::GetUnaggregatedReceipts).unwrap(); // Check that the allocation's unaggregated fees have *not* increased. assert_eq!( @@ -384,14 +410,16 @@ mod tests { value: 20, }; allocation - .cast(SenderAllocationMsg::NewReceipt(new_receipt_notification)) + .cast(SenderAllocationMessage::NewReceipt( + new_receipt_notification, + )) .unwrap(); ractor::concurrency::sleep(Duration::from_millis(10)).await; expected_unaggregated_fees += 20; let allocation_unaggregated_fees = - call!(allocation, SenderAllocationMsg::GetUnaggregatedReceipts).unwrap(); + call!(allocation, SenderAllocationMessage::GetUnaggregatedReceipts).unwrap(); // Check that the allocation's unaggregated fees are correct. assert_eq!( @@ -415,13 +443,15 @@ mod tests { value: 19, }; allocation - .cast(SenderAllocationMsg::NewReceipt(new_receipt_notification)) + .cast(SenderAllocationMessage::NewReceipt( + new_receipt_notification, + )) .unwrap(); ractor::concurrency::sleep(Duration::from_millis(10)).await; let allocation_unaggregated_fees = - call!(allocation, SenderAllocationMsg::GetUnaggregatedReceipts).unwrap(); + call!(allocation, SenderAllocationMessage::GetUnaggregatedReceipts).unwrap(); // Check that the allocation's unaggregated fees have *not* increased. assert_eq!( @@ -435,6 +465,9 @@ mod tests { allocation_tracker.get_total_fee(), expected_unaggregated_fees ); + + sender.stop(None); + handle.await.unwrap(); } #[sqlx::test(migrations = "../migrations")] @@ -468,15 +501,15 @@ mod tests { .await; // Create a sender_account. - let sender_account = create_sender_with_allocations( + let (sender_account, sender_handle, prefix) = create_sender_with_allocations( pgpool.clone(), "http://".to_owned() + &aggregator_endpoint.to_string(), &mock_server.uri(), ) .await; - let allocation = ActorRef::::where_is(format!( - "{sender}:{allocation_id}", + let allocation = ActorRef::::where_is(format!( + "{prefix}:{sender}:{allocation_id}", sender = SENDER.1, allocation_id = *ALLOCATION_ID_0 )) @@ -504,7 +537,9 @@ mod tests { value, }; allocation - .cast(SenderAllocationMsg::NewReceipt(new_receipt_notification)) + .cast(SenderAllocationMessage::NewReceipt( + new_receipt_notification, + )) .unwrap(); ractor::concurrency::sleep(Duration::from_millis(100)).await; @@ -564,7 +599,7 @@ mod tests { // Check that the allocation's unaggregated fees value is reduced. let allocation_unaggregated_fees = - call!(allocation, SenderAllocationMsg::GetUnaggregatedReceipts).unwrap(); + call!(allocation, SenderAllocationMessage::GetUnaggregatedReceipts).unwrap(); // Check that the allocation's unaggregated fees have *not* increased. assert!(allocation_unaggregated_fees.value <= trigger_value); @@ -596,7 +631,9 @@ mod tests { value, }; allocation - .cast(SenderAllocationMsg::NewReceipt(new_receipt_notification)) + .cast(SenderAllocationMessage::NewReceipt( + new_receipt_notification, + )) .unwrap(); ractor::concurrency::sleep(Duration::from_millis(10)).await; @@ -655,7 +692,7 @@ mod tests { // Check that the allocation's unaggregated fees value is reduced. let allocation_unaggregated_fees = - call!(allocation, SenderAllocationMsg::GetUnaggregatedReceipts).unwrap(); + call!(allocation, SenderAllocationMessage::GetUnaggregatedReceipts).unwrap(); // Check that the allocation's unaggregated fees have *not* increased. assert!(allocation_unaggregated_fees.value <= trigger_value); @@ -668,40 +705,51 @@ mod tests { // Stop the TAP aggregator server. handle.stop().unwrap(); handle.stopped().await; + + sender_account.stop(None); + sender_handle.await.unwrap(); } #[sqlx::test(migrations = "../migrations")] async fn test_sender_unaggregated_fees(pgpool: PgPool) { // Create a sender_account. - let sender_account = + let (sender_account, handle, _prefix) = create_sender_with_allocations(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL).await; + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + // Closure that adds a number of receipts to an allocation. - let update_receipt_fees = |allocation_id: Address, value: u128| { - let sender_account = sender_account.clone(); - sender_account - .cast(SenderAccountMessage::UpdateReceiptFees( + + let update_receipt_fees = |sender_account: &ActorRef, + allocation_id: Address, + value: u128| { + cast!( + sender_account, + SenderAccountMessage::UpdateReceiptFees( allocation_id, UnaggregatedReceipts { value, ..Default::default() }, - )) - .unwrap(); + ) + ) + .unwrap(); value }; + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + // Add receipts to the database for allocation_0 - let total_value_0 = update_receipt_fees(*ALLOCATION_ID_0, 90); + let total_value_0 = update_receipt_fees(&sender_account, *ALLOCATION_ID_0, 90); tokio::time::sleep(std::time::Duration::from_millis(10)).await; // Add receipts to the database for allocation_1 - let total_value_1 = update_receipt_fees(*ALLOCATION_ID_1, 100); + let total_value_1 = update_receipt_fees(&sender_account, *ALLOCATION_ID_1, 100); tokio::time::sleep(std::time::Duration::from_millis(10)).await; // Add receipts to the database for allocation_2 - let total_value_2 = update_receipt_fees(*ALLOCATION_ID_2, 80); + let total_value_2 = update_receipt_fees(&sender_account, *ALLOCATION_ID_2, 80); tokio::time::sleep(std::time::Duration::from_millis(10)).await; let allocation_tracker = @@ -718,5 +766,8 @@ mod tests { allocation_tracker.get_total_fee(), total_value_0 + total_value_1 + total_value_2 ); + + sender_account.stop(None); + handle.await.unwrap(); } } diff --git a/tap-agent/src/agent/sender_accounts_manager.rs b/tap-agent/src/agent/sender_accounts_manager.rs index d6f745367..1eb3c6ce3 100644 --- a/tap-agent/src/agent/sender_accounts_manager.rs +++ b/tap-agent/src/agent/sender_accounts_manager.rs @@ -4,7 +4,7 @@ use std::collections::HashSet; use std::{collections::HashMap, str::FromStr}; -use crate::agent::sender_allocation::SenderAllocationMsg; +use crate::agent::sender_allocation::SenderAllocationMessage; use alloy_sol_types::Eip712Domain; use anyhow::anyhow; use anyhow::Result; @@ -175,7 +175,7 @@ impl Actor for SenderAccountsManager { SenderAccount::spawn_linked( Some(sender_id.to_string()), sender_account, - allocation_ids, + (allocation_ids, None), myself.get_cell(), ) .await?; @@ -344,12 +344,12 @@ async fn new_receipts_watcher( }; let allocation_id = &new_receipt_notification.allocation_id; - if let Some(sender_allocation) = - ActorRef::::where_is(format!("{sender_address}:{allocation_id}")) - { - if let Err(e) = - sender_allocation.cast(SenderAllocationMsg::NewReceipt(new_receipt_notification)) - { + if let Some(sender_allocation) = ActorRef::::where_is(format!( + "{sender_address}:{allocation_id}" + )) { + if let Err(e) = sender_allocation.cast(SenderAllocationMessage::NewReceipt( + new_receipt_notification, + )) { error!( "Error while forwarding new receipt notification to sender_allocation: {:?}", e @@ -489,7 +489,7 @@ mod tests { let sender_allocation_id = format!("{}:{}", SENDER.1, allocation_id); let sender_allocation = - ActorRef::::where_is(sender_allocation_id).unwrap(); + ActorRef::::where_is(sender_allocation_id).unwrap(); assert_eq!(sender_allocation.get_status(), ActorStatus::Running); // Remove the escrow account from the escrow_accounts Eventual. diff --git a/tap-agent/src/agent/sender_allocation.rs b/tap-agent/src/agent/sender_allocation.rs index 332881380..daf24d708 100644 --- a/tap-agent/src/agent/sender_allocation.rs +++ b/tap-agent/src/agent/sender_allocation.rs @@ -49,16 +49,18 @@ pub struct SenderAllocation { sender_account_ref: ActorRef, } -pub enum SenderAllocationMsg { +pub enum SenderAllocationMessage { NewReceipt(NewReceiptNotification), TriggerRAVRequest(RpcReplyPort), - GetUnaggregatedReceipts(RpcReplyPort), CloseAllocation, + + #[cfg(test)] + GetUnaggregatedReceipts(RpcReplyPort), } #[async_trait::async_trait] impl Actor for SenderAllocation { - type Msg = SenderAllocationMsg; + type Msg = SenderAllocationMessage; type State = UnaggregatedReceipts; type Arguments = (); @@ -98,7 +100,7 @@ impl Actor for SenderAllocation { state: &mut Self::State, ) -> std::result::Result<(), ActorProcessingErr> { match message { - SenderAllocationMsg::NewReceipt(NewReceiptNotification { + SenderAllocationMessage::NewReceipt(NewReceiptNotification { id, value: fees, .. }) => { if id > state.last_id { @@ -120,7 +122,7 @@ impl Actor for SenderAllocation { ))?; } } - SenderAllocationMsg::TriggerRAVRequest(reply) => { + SenderAllocationMessage::TriggerRAVRequest(reply) => { self.rav_requester_single().await.map_err(|e| { anyhow! { "Error while requesting RAV for sender {} and allocation {}: {}", @@ -135,12 +137,7 @@ impl Actor for SenderAllocation { } } - SenderAllocationMsg::GetUnaggregatedReceipts(reply) => { - if !reply.is_closed() { - let _ = reply.send(state.clone()); - } - } - SenderAllocationMsg::CloseAllocation => { + SenderAllocationMessage::CloseAllocation => { self.rav_requester_single().await.inspect_err(|e| { error!( "Error while requesting RAV for sender {} and allocation {}: {}", @@ -155,6 +152,13 @@ impl Actor for SenderAllocation { })?; myself.stop(None); } + + #[cfg(test)] + SenderAllocationMessage::GetUnaggregatedReceipts(reply) => { + if !reply.is_closed() { + let _ = reply.send(state.clone()); + } + } } Ok(()) } @@ -458,192 +462,229 @@ impl SenderAllocation { #[cfg(test)] mod tests { - // - // use std::collections::HashMap; - // - // use indexer_common::subgraph_client::DeploymentDetails; - // use serde_json::json; - // use tap_aggregator::server::run_server; - // - // use wiremock::{ - // matchers::{body_string_contains, method}, - // Mock, MockServer, ResponseTemplate, - // }; - // - // use super::*; - // use crate::tap::test_utils::{ - // create_rav, create_received_receipt, store_rav, store_receipt, ALLOCATION_ID_0, INDEXER, - // SENDER, SIGNER, TAP_EIP712_DOMAIN_SEPARATOR, - // }; - // - // const DUMMY_URL: &str = "http://localhost:1234"; - // - // async fn create_sender_allocation( - // pgpool: PgPool, - // sender_aggregator_endpoint: String, - // escrow_subgraph_endpoint: &str, - // ) -> SenderAllocation { - // let config = Box::leak(Box::new(config::Cli { - // config: None, - // ethereum: config::Ethereum { - // indexer_address: INDEXER.1, - // }, - // tap: config::Tap { - // rav_request_trigger_value: 100, - // rav_request_timestamp_buffer_ms: 1, - // rav_request_timeout_secs: 5, - // ..Default::default() - // }, - // ..Default::default() - // })); - // - // let escrow_subgraph = Box::leak(Box::new(SubgraphClient::new( - // reqwest::Client::new(), - // None, - // DeploymentDetails::for_query_url(escrow_subgraph_endpoint).unwrap(), - // ))); - // - // let escrow_accounts_eventual = Eventual::from_value(EscrowAccounts::new( - // HashMap::from([(SENDER.1, 1000.into())]), - // HashMap::from([(SENDER.1, vec![SIGNER.1])]), - // )); - // - // let escrow_adapter = EscrowAdapter::new(escrow_accounts_eventual.clone(), SENDER.1); - // - // SenderAllocation::new( - // config, - // pgpool.clone(), - // *ALLOCATION_ID_0, - // SENDER.1, - // escrow_accounts_eventual, - // escrow_subgraph, - // escrow_adapter, - // TAP_EIP712_DOMAIN_SEPARATOR.clone(), - // sender_aggregator_endpoint, - // ) - // .await - // } - // - // /// Test that the sender_allocation correctly updates the unaggregated fees from the - // /// database when there is no RAV in the database. - // /// - // /// The sender_allocation should consider all receipts found for the allocation and - // /// sender. - // #[sqlx::test(migrations = "../migrations")] - // async fn test_update_unaggregated_fees_no_rav(pgpool: PgPool) { - // let sender_allocation = - // create_sender_allocation(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL).await; - // - // // Add receipts to the database. - // for i in 1..10 { - // let receipt = - // create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i, i.into()).await; - // store_receipt(&pgpool, receipt.signed_receipt()) - // .await - // .unwrap(); - // } - // - // // Let the sender_allocation update the unaggregated fees from the database. - // sender_allocation.update_unaggregated_fees().await.unwrap(); - // - // // Check that the unaggregated fees are correct. - // assert_eq!( - // sender_allocation.unaggregated_fees.lock().unwrap().value, - // 45u128 - // ); - // } - // - // /// Test that the sender_allocation correctly updates the unaggregated fees from the - // /// database when there is a RAV in the database as well as receipts which timestamp are lesser - // /// and greater than the RAV's timestamp. - // /// - // /// The sender_allocation should only consider receipts with a timestamp greater - // /// than the RAV's timestamp. - // #[sqlx::test(migrations = "../migrations")] - // async fn test_update_unaggregated_fees_with_rav(pgpool: PgPool) { - // let sender_allocation = - // create_sender_allocation(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL).await; - // - // // Add the RAV to the database. - // // This RAV has timestamp 4. The sender_allocation should only consider receipts - // // with a timestamp greater than 4. - // let signed_rav = create_rav(*ALLOCATION_ID_0, SIGNER.0.clone(), 4, 10).await; - // store_rav(&pgpool, signed_rav, SENDER.1).await.unwrap(); - // - // // Add receipts to the database. - // for i in 1..10 { - // let receipt = - // create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i, i.into()).await; - // store_receipt(&pgpool, receipt.signed_receipt()) - // .await - // .unwrap(); - // } - // - // // Let the sender_allocation update the unaggregated fees from the database. - // sender_allocation.update_unaggregated_fees().await.unwrap(); - // - // // Check that the unaggregated fees are correct. - // assert_eq!( - // sender_allocation.unaggregated_fees.lock().unwrap().value, - // 35u128 - // ); - // } - // - // #[sqlx::test(migrations = "../migrations")] - // async fn test_rav_requester_manual(pgpool: PgPool) { - // // Start a TAP aggregator server. - // let (handle, aggregator_endpoint) = run_server( - // 0, - // SIGNER.0.clone(), - // vec![SIGNER.1].into_iter().collect(), - // TAP_EIP712_DOMAIN_SEPARATOR.clone(), - // 100 * 1024, - // 100 * 1024, - // 1, - // ) - // .await - // .unwrap(); - // - // // Start a mock graphql server using wiremock - // let mock_server = MockServer::start().await; - // - // // Mock result for TAP redeem txs for (allocation, sender) pair. - // mock_server - // .register( - // Mock::given(method("POST")) - // .and(body_string_contains("transactions")) - // .respond_with( - // ResponseTemplate::new(200) - // .set_body_json(json!({ "data": { "transactions": []}})), - // ), - // ) - // .await; - // - // // Create a sender_allocation. - // let sender_allocation = create_sender_allocation( - // pgpool.clone(), - // "http://".to_owned() + &aggregator_endpoint.to_string(), - // &mock_server.uri(), - // ) - // .await; - // - // // Add receipts to the database. - // for i in 0..10 { - // let receipt = - // create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i + 1, i.into()).await; - // store_receipt(&pgpool, receipt.signed_receipt()) - // .await - // .unwrap(); - // } - // - // // Let the sender_allocation update the unaggregated fees from the database. - // sender_allocation.update_unaggregated_fees().await.unwrap(); - // - // // Trigger a RAV request manually. - // sender_allocation.rav_requester_single().await.unwrap(); - // - // // Stop the TAP aggregator server. - // handle.stop().unwrap(); - // handle.stopped().await; - // } + + use std::collections::HashMap; + + use indexer_common::subgraph_client::DeploymentDetails; + use ractor::call; + use serde_json::json; + use tap_aggregator::server::run_server; + + use wiremock::{ + matchers::{body_string_contains, method}, + Mock, MockServer, ResponseTemplate, + }; + + struct MockSenderAccount; + + #[async_trait::async_trait] + impl Actor for MockSenderAccount { + type Msg = SenderAccountMessage; + type State = (); + type Arguments = (); + + async fn pre_start( + &self, + _myself: ActorRef, + _allocation_ids: Self::Arguments, + ) -> std::result::Result { + Ok(()) + } + } + + use super::*; + use crate::tap::test_utils::{ + create_rav, create_received_receipt, store_rav, store_receipt, ALLOCATION_ID_0, INDEXER, + SENDER, SIGNER, TAP_EIP712_DOMAIN_SEPARATOR, + }; + + const DUMMY_URL: &str = "http://localhost:1234"; + + async fn create_sender_allocation( + pgpool: PgPool, + sender_aggregator_endpoint: String, + escrow_subgraph_endpoint: &str, + ) -> ActorRef { + let config = Box::leak(Box::new(config::Cli { + config: None, + ethereum: config::Ethereum { + indexer_address: INDEXER.1, + }, + tap: config::Tap { + rav_request_trigger_value: 100, + rav_request_timestamp_buffer_ms: 1, + rav_request_timeout_secs: 5, + ..Default::default() + }, + ..Default::default() + })); + + let escrow_subgraph = Box::leak(Box::new(SubgraphClient::new( + reqwest::Client::new(), + None, + DeploymentDetails::for_query_url(escrow_subgraph_endpoint).unwrap(), + ))); + + let escrow_accounts_eventual = Eventual::from_value(EscrowAccounts::new( + HashMap::from([(SENDER.1, 1000.into())]), + HashMap::from([(SENDER.1, vec![SIGNER.1])]), + )); + + let escrow_adapter = EscrowAdapter::new(escrow_accounts_eventual.clone(), SENDER.1); + + let (sender_account_ref, _join_handle) = + MockSenderAccount::spawn(None, MockSenderAccount, ()) + .await + .unwrap(); + + let allocation = SenderAllocation::new( + config, + pgpool.clone(), + *ALLOCATION_ID_0, + SENDER.1, + escrow_accounts_eventual, + escrow_subgraph, + escrow_adapter, + TAP_EIP712_DOMAIN_SEPARATOR.clone(), + sender_aggregator_endpoint, + sender_account_ref, + ) + .await; + + let (allocation_ref, _join_handle) = + SenderAllocation::spawn(None, allocation, ()).await.unwrap(); + + allocation_ref + } + + /// Test that the sender_allocation correctly updates the unaggregated fees from the + /// database when there is no RAV in the database. + /// + /// The sender_allocation should consider all receipts found for the allocation and + /// sender. + #[sqlx::test(migrations = "../migrations")] + async fn test_update_unaggregated_fees_no_rav(pgpool: PgPool) { + // Add receipts to the database. + for i in 1..10 { + let receipt = + create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i, i.into()).await; + store_receipt(&pgpool, receipt.signed_receipt()) + .await + .unwrap(); + } + + let sender_allocation = + create_sender_allocation(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL).await; + + // Get total_unaggregated_fees + let total_unaggregated_fees = call!( + sender_allocation, + SenderAllocationMessage::GetUnaggregatedReceipts + ) + .unwrap(); + + // Check that the unaggregated fees are correct. + assert_eq!(total_unaggregated_fees.value, 45u128); + } + + /// Test that the sender_allocation correctly updates the unaggregated fees from the + /// database when there is a RAV in the database as well as receipts which timestamp are lesser + /// and greater than the RAV's timestamp. + /// + /// The sender_allocation should only consider receipts with a timestamp greater + /// than the RAV's timestamp. + #[sqlx::test(migrations = "../migrations")] + async fn test_update_unaggregated_fees_with_rav(pgpool: PgPool) { + // Add the RAV to the database. + // This RAV has timestamp 4. The sender_allocation should only consider receipts + // with a timestamp greater than 4. + let signed_rav = create_rav(*ALLOCATION_ID_0, SIGNER.0.clone(), 4, 10).await; + store_rav(&pgpool, signed_rav, SENDER.1).await.unwrap(); + + // Add receipts to the database. + for i in 1..10 { + let receipt = + create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i, i.into()).await; + store_receipt(&pgpool, receipt.signed_receipt()) + .await + .unwrap(); + } + + let sender_allocation = + create_sender_allocation(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL).await; + + // Get total_unaggregated_fees + let total_unaggregated_fees = call!( + sender_allocation, + SenderAllocationMessage::GetUnaggregatedReceipts + ) + .unwrap(); + + // Check that the unaggregated fees are correct. + assert_eq!(total_unaggregated_fees.value, 35u128); + } + + #[sqlx::test(migrations = "../migrations")] + async fn test_rav_requester_manual(pgpool: PgPool) { + // Start a TAP aggregator server. + let (handle, aggregator_endpoint) = run_server( + 0, + SIGNER.0.clone(), + vec![SIGNER.1].into_iter().collect(), + TAP_EIP712_DOMAIN_SEPARATOR.clone(), + 100 * 1024, + 100 * 1024, + 1, + ) + .await + .unwrap(); + + // Start a mock graphql server using wiremock + let mock_server = MockServer::start().await; + + // Mock result for TAP redeem txs for (allocation, sender) pair. + mock_server + .register( + Mock::given(method("POST")) + .and(body_string_contains("transactions")) + .respond_with( + ResponseTemplate::new(200) + .set_body_json(json!({ "data": { "transactions": []}})), + ), + ) + .await; + + // Add receipts to the database. + for i in 0..10 { + let receipt = + create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i + 1, i.into()).await; + store_receipt(&pgpool, receipt.signed_receipt()) + .await + .unwrap(); + } + + // Create a sender_allocation. + let sender_allocation = create_sender_allocation( + pgpool.clone(), + "http://".to_owned() + &aggregator_endpoint.to_string(), + &mock_server.uri(), + ) + .await; + + // Trigger a RAV request manually. + + // Get total_unaggregated_fees + let total_unaggregated_fees = call!( + sender_allocation, + SenderAllocationMessage::TriggerRAVRequest + ) + .unwrap(); + + // Check that the unaggregated fees are correct. + assert_eq!(total_unaggregated_fees.value, 0u128); + + // Stop the TAP aggregator server. + handle.stop().unwrap(); + handle.stopped().await; + } } From ac8aa50fff3d2742e5cf7df10ffa2aa83f71c99d Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Mon, 1 Apr 2024 20:41:39 -0300 Subject: [PATCH 14/41] refactor: use args and state instead of actor fields Signed-off-by: Gustavo Inacio --- tap-agent/src/agent/sender_account.rs | 152 ++++++------- .../src/agent/sender_accounts_manager.rs | 37 +-- tap-agent/src/agent/sender_allocation.rs | 215 ++++++++++-------- 3 files changed, 214 insertions(+), 190 deletions(-) diff --git a/tap-agent/src/agent/sender_account.rs b/tap-agent/src/agent/sender_account.rs index c3b61f33d..50e9e94f7 100644 --- a/tap-agent/src/agent/sender_account.rs +++ b/tap-agent/src/agent/sender_account.rs @@ -12,7 +12,7 @@ use sqlx::PgPool; use thegraph::types::Address; use tracing::error; -use super::sender_allocation::SenderAllocation; +use super::sender_allocation::{SenderAllocation, SenderAllocationArgs}; use crate::agent::allocation_id_tracker::AllocationIdTracker; use crate::agent::sender_allocation::SenderAllocationMessage; use crate::agent::unaggregated_receipts::UnaggregatedReceipts; @@ -39,26 +39,37 @@ pub enum SenderAccountMessage { /// - Requesting RAVs from the sender's TAP aggregator once the cumulative unaggregated fees reach a /// certain threshold. /// - Requesting the last RAV from the sender's TAP aggregator for all EOL allocations. -pub struct SenderAccount { +pub struct SenderAccount; + +pub struct SenderAccountArgs { + pub config: &'static config::Cli, + pub pgpool: PgPool, + pub sender_id: Address, + pub escrow_accounts: Eventual, + pub indexer_allocations: Eventual>, + pub escrow_subgraph: &'static SubgraphClient, + pub domain_separator: Eip712Domain, + pub sender_aggregator_endpoint: String, + pub allocation_ids: HashSet
, + pub prefix: Option, +} +pub struct State { + prefix: Option, + allocation_id_tracker: AllocationIdTracker, + allocation_ids: HashSet
, + _indexer_allocations_handle: PipeHandle, + sender: Address, + //Eventuals escrow_accounts: Eventual, - indexer_allocations: Eventual>, escrow_subgraph: &'static SubgraphClient, escrow_adapter: EscrowAdapter, - tap_eip712_domain_separator: Eip712Domain, + domain_separator: Eip712Domain, config: &'static config::Cli, pgpool: PgPool, - sender: Address, sender_aggregator_endpoint: String, } -pub struct State { - prefix: Option, - allocation_id_tracker: AllocationIdTracker, - allocation_ids: HashSet
, - _indexer_allocations_handle: PipeHandle, - sender: Address, -} impl State { fn format_sender_allocation(&self, allocation_id: &Address) -> String { @@ -94,16 +105,27 @@ impl State { impl Actor for SenderAccount { type Msg = SenderAccountMessage; type State = State; - type Arguments = (HashSet
, Option); + type Arguments = SenderAccountArgs; async fn pre_start( &self, myself: ActorRef, - (allocation_ids, prefix): Self::Arguments, + SenderAccountArgs { + config, + pgpool, + sender_id, + escrow_accounts, + indexer_allocations, + escrow_subgraph, + domain_separator, + sender_aggregator_endpoint, + allocation_ids, + prefix, + }: Self::Arguments, ) -> std::result::Result { let clone = myself.clone(); let _indexer_allocations_handle = - self.indexer_allocations + indexer_allocations .clone() .pipe_async(move |allocation_ids| { let myself = clone.clone(); @@ -122,12 +144,21 @@ impl Actor for SenderAccount { myself.cast(SenderAccountMessage::CreateSenderAllocation(*allocation_id))?; } + let escrow_adapter = EscrowAdapter::new(escrow_accounts.clone(), sender_id); + Ok(State { allocation_id_tracker: AllocationIdTracker::new(), allocation_ids, _indexer_allocations_handle, prefix, - sender: self.sender, + escrow_accounts, + escrow_subgraph, + escrow_adapter, + domain_separator, + sender_aggregator_endpoint, + config, + pgpool, + sender: sender_id, }) } @@ -143,29 +174,28 @@ impl Actor for SenderAccount { let tracker = &mut state.allocation_id_tracker; tracker.add_or_update(allocation_id, unaggregated_fees.value); - if tracker.get_total_fee() >= self.config.tap.rav_request_trigger_value.into() { + if tracker.get_total_fee() >= state.config.tap.rav_request_trigger_value.into() { state.rav_requester_single().await?; } } SenderAccountMessage::CreateSenderAllocation(allocation_id) => { - let sender_allocation = SenderAllocation::new( - self.config, - self.pgpool.clone(), + let args = SenderAllocationArgs { + config: state.config, + pgpool: state.pgpool.clone(), allocation_id, - self.sender, - self.escrow_accounts.clone(), - self.escrow_subgraph, - self.escrow_adapter.clone(), - self.tap_eip712_domain_separator.clone(), - self.sender_aggregator_endpoint.clone(), - myself.clone(), - ) - .await; + sender: state.sender, + escrow_accounts: state.escrow_accounts.clone(), + escrow_subgraph: state.escrow_subgraph, + escrow_adapter: state.escrow_adapter.clone(), + domain_separator: state.domain_separator.clone(), + sender_aggregator_endpoint: state.sender_aggregator_endpoint.clone(), + sender_account_ref: myself.clone(), + }; SenderAllocation::spawn_linked( Some(state.format_sender_allocation(&allocation_id)), - sender_allocation, - (), + SenderAllocation, + args, myself.get_cell(), ) .await?; @@ -198,34 +228,6 @@ impl Actor for SenderAccount { } } -impl SenderAccount { - #[allow(clippy::too_many_arguments)] - pub fn new( - config: &'static config::Cli, - pgpool: PgPool, - sender_id: Address, - escrow_accounts: Eventual, - indexer_allocations: Eventual>, - escrow_subgraph: &'static SubgraphClient, - tap_eip712_domain_separator: Eip712Domain, - sender_aggregator_endpoint: String, - ) -> Self { - let escrow_adapter = EscrowAdapter::new(escrow_accounts.clone(), sender_id); - - Self { - escrow_accounts, - indexer_allocations, - escrow_subgraph, - escrow_adapter, - tap_eip712_domain_separator, - sender_aggregator_endpoint, - config, - pgpool, - sender: sender_id, - } - } -} - #[cfg(test)] mod tests { use crate::agent::sender_accounts_manager::NewReceiptNotification; @@ -295,29 +297,27 @@ mod tests { *ALLOCATION_ID_2, ])); - let sender = SenderAccount::new( + let prefix = format!( + "test-{}", + PREFIX_ID.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + ); + + let args = SenderAccountArgs { config, pgpool, - SENDER.1, - escrow_accounts_eventual, + sender_id: SENDER.1, + escrow_accounts: escrow_accounts_eventual, indexer_allocations, escrow_subgraph, - TAP_EIP712_DOMAIN_SEPARATOR.clone(), + domain_separator: TAP_EIP712_DOMAIN_SEPARATOR.clone(), sender_aggregator_endpoint, - ); - - let prefix = format!( - "test-{}", - PREFIX_ID.fetch_add(1, std::sync::atomic::Ordering::SeqCst) - ); + allocation_ids: HashSet::new(), + prefix: Some(prefix.clone()), + }; - let (sender, handle) = SenderAccount::spawn( - Some(prefix.clone()), - sender, - (HashSet::new(), Some(prefix.clone())), - ) - .await - .unwrap(); + let (sender, handle) = SenderAccount::spawn(Some(prefix.clone()), SenderAccount, args) + .await + .unwrap(); // await for the allocations to be created ractor::concurrency::sleep(Duration::from_millis(100)).await; diff --git a/tap-agent/src/agent/sender_accounts_manager.rs b/tap-agent/src/agent/sender_accounts_manager.rs index 1eb3c6ce3..03892c7ff 100644 --- a/tap-agent/src/agent/sender_accounts_manager.rs +++ b/tap-agent/src/agent/sender_accounts_manager.rs @@ -17,7 +17,7 @@ use sqlx::{postgres::PgListener, PgPool}; use thegraph::types::Address; use tracing::{error, warn}; -use super::sender_account::{SenderAccount, SenderAccountMessage}; +use super::sender_account::{SenderAccount, SenderAccountArgs, SenderAccountMessage}; use crate::config; #[derive(Deserialize, Debug)] @@ -171,11 +171,11 @@ impl Actor for SenderAccountsManager { state.sender_ids = target_senders; } SenderAccountsManagerMessage::CreateSenderAccount(sender_id, allocation_ids) => { - let sender_account = state.new_sender_account(&sender_id)?; + let args = state.new_sender_account_args(&sender_id, allocation_ids)?; SenderAccount::spawn_linked( Some(sender_id.to_string()), - sender_account, - (allocation_ids, None), + SenderAccount, + args, myself.get_cell(), ) .await?; @@ -284,16 +284,21 @@ impl State { } unfinalized_sender_allocations_map } - fn new_sender_account(&self, sender_id: &Address) -> Result { - Ok(SenderAccount::new( - self.config, - self.pgpool.clone(), - *sender_id, - self.escrow_accounts.clone(), - self.indexer_allocations.clone(), - self.escrow_subgraph, - self.domain_separator.clone(), - self.sender_aggregator_endpoints + fn new_sender_account_args( + &self, + sender_id: &Address, + allocation_ids: HashSet
, + ) -> Result { + Ok(SenderAccountArgs { + config: self.config, + pgpool: self.pgpool.clone(), + sender_id: *sender_id, + escrow_accounts: self.escrow_accounts.clone(), + indexer_allocations: self.indexer_allocations.clone(), + escrow_subgraph: self.escrow_subgraph, + domain_separator: self.domain_separator.clone(), + sender_aggregator_endpoint: self + .sender_aggregator_endpoints .get(sender_id) .ok_or_else(|| { anyhow!( @@ -302,7 +307,9 @@ impl State { ) })? .clone(), - )) + allocation_ids, + prefix: None, + }) } } diff --git a/tap-agent/src/agent/sender_allocation.rs b/tap-agent/src/agent/sender_allocation.rs index daf24d708..7b44cbd74 100644 --- a/tap-agent/src/agent/sender_allocation.rs +++ b/tap-agent/src/agent/sender_allocation.rs @@ -37,7 +37,10 @@ use crate::{ type TapManager = tap_core::manager::Manager; /// Manages unaggregated fees and the TAP lifecyle for a specific (allocation, sender) pair. -pub struct SenderAllocation { +pub struct SenderAllocation; + +pub struct SenderAllocationState { + unaggregated_fees: UnaggregatedReceipts, pgpool: PgPool, tap_manager: TapManager, allocation_id: Address, @@ -45,10 +48,23 @@ pub struct SenderAllocation { sender_aggregator_endpoint: String, config: &'static config::Cli, escrow_accounts: Eventual, - tap_eip712_domain_separator: Eip712Domain, + domain_separator: Eip712Domain, sender_account_ref: ActorRef, } +pub struct SenderAllocationArgs { + pub config: &'static config::Cli, + pub pgpool: PgPool, + pub allocation_id: Address, + pub sender: Address, + pub escrow_accounts: Eventual, + pub escrow_subgraph: &'static SubgraphClient, + pub escrow_adapter: EscrowAdapter, + pub domain_separator: Eip712Domain, + pub sender_aggregator_endpoint: String, + pub sender_account_ref: ActorRef, +} + pub enum SenderAllocationMessage { NewReceipt(NewReceiptNotification), TriggerRAVRequest(RpcReplyPort), @@ -61,33 +77,82 @@ pub enum SenderAllocationMessage { #[async_trait::async_trait] impl Actor for SenderAllocation { type Msg = SenderAllocationMessage; - type State = UnaggregatedReceipts; - type Arguments = (); + type State = SenderAllocationState; + type Arguments = SenderAllocationArgs; async fn pre_start( &self, _myself: ActorRef, - _args: Self::Arguments, + SenderAllocationArgs { + config, + pgpool, + allocation_id, + sender, + escrow_accounts, + escrow_subgraph, + escrow_adapter, + domain_separator, + sender_aggregator_endpoint, + sender_account_ref, + }: Self::Arguments, ) -> std::result::Result { - let unaggregated_fees = self.calculate_unaggregated_fee().await?; - self.sender_account_ref - .cast(SenderAccountMessage::UpdateReceiptFees( - self.allocation_id, - unaggregated_fees.clone(), - ))?; + let required_checks: Vec> = vec![ + Arc::new(AllocationId::new( + sender, + allocation_id, + escrow_subgraph, + config, + )), + Arc::new(Signature::new( + domain_separator.clone(), + escrow_accounts.clone(), + )), + ]; + let context = TapAgentContext::new( + pgpool.clone(), + allocation_id, + sender, + escrow_accounts.clone(), + escrow_adapter, + ); + let tap_manager = TapManager::new( + domain_separator.clone(), + context, + Checks::new(required_checks), + ); + + let mut state = SenderAllocationState { + pgpool, + tap_manager, + allocation_id, + sender, + sender_aggregator_endpoint, + config, + escrow_accounts, + domain_separator, + sender_account_ref: sender_account_ref.clone(), + unaggregated_fees: UnaggregatedReceipts::default(), + }; - Ok(unaggregated_fees) + state.unaggregated_fees = state.calculate_unaggregated_fee().await?; + sender_account_ref.cast(SenderAccountMessage::UpdateReceiptFees( + allocation_id, + state.unaggregated_fees.clone(), + ))?; + + Ok(state) } async fn post_stop( &self, _myself: ActorRef, - _state: &mut Self::State, + state: &mut Self::State, ) -> std::result::Result<(), ActorProcessingErr> { // cleanup receipt fees for sender - self.sender_account_ref + state + .sender_account_ref .cast(SenderAccountMessage::UpdateReceiptFees( - self.allocation_id, + state.allocation_id, UnaggregatedReceipts::default(), ))?; Ok(()) @@ -99,55 +164,58 @@ impl Actor for SenderAllocation { message: Self::Msg, state: &mut Self::State, ) -> std::result::Result<(), ActorProcessingErr> { + let unaggreated_fees = &mut state.unaggregated_fees; match message { SenderAllocationMessage::NewReceipt(NewReceiptNotification { id, value: fees, .. }) => { - if id > state.last_id { - state.last_id = id; - state.value = state.value.checked_add(fees).unwrap_or_else(|| { - // This should never happen, but if it does, we want to know about it. - error!( + if id > unaggreated_fees.last_id { + unaggreated_fees.last_id = id; + unaggreated_fees.value = + unaggreated_fees.value.checked_add(fees).unwrap_or_else(|| { + // This should never happen, but if it does, we want to know about it. + error!( "Overflow when adding receipt value {} to total unaggregated fees {} \ for allocation {} and sender {}. Setting total unaggregated fees to \ u128::MAX.", - fees, state.value, self.allocation_id, self.sender + fees, unaggreated_fees.value, state.allocation_id, state.sender ); - u128::MAX - }); - self.sender_account_ref + u128::MAX + }); + state + .sender_account_ref .cast(SenderAccountMessage::UpdateReceiptFees( - self.allocation_id, - state.clone(), + state.allocation_id, + unaggreated_fees.clone(), ))?; } } SenderAllocationMessage::TriggerRAVRequest(reply) => { - self.rav_requester_single().await.map_err(|e| { + state.rav_requester_single().await.map_err(|e| { anyhow! { "Error while requesting RAV for sender {} and allocation {}: {}", - self.sender, - self.allocation_id, + state.sender, + state.allocation_id, e } })?; - *state = self.calculate_unaggregated_fee().await?; + state.unaggregated_fees = state.calculate_unaggregated_fee().await?; if !reply.is_closed() { - let _ = reply.send(state.clone()); + let _ = reply.send(state.unaggregated_fees.clone()); } } SenderAllocationMessage::CloseAllocation => { - self.rav_requester_single().await.inspect_err(|e| { + state.rav_requester_single().await.inspect_err(|e| { error!( "Error while requesting RAV for sender {} and allocation {}: {}", - self.sender, self.allocation_id, e + state.sender, state.allocation_id, e ); })?; - self.mark_rav_final().await.inspect_err(|e| { + state.mark_rav_final().await.inspect_err(|e| { error!( "Error while marking allocation {} as final for sender {}: {}", - self.allocation_id, self.sender, e + state.allocation_id, state.sender, e ); })?; myself.stop(None); @@ -156,7 +224,7 @@ impl Actor for SenderAllocation { #[cfg(test)] SenderAllocationMessage::GetUnaggregatedReceipts(reply) => { if !reply.is_closed() { - let _ = reply.send(state.clone()); + let _ = reply.send(unaggreated_fees.clone()); } } } @@ -164,58 +232,7 @@ impl Actor for SenderAllocation { } } -impl SenderAllocation { - #[allow(clippy::too_many_arguments)] - pub async fn new( - config: &'static config::Cli, - pgpool: PgPool, - allocation_id: Address, - sender: Address, - escrow_accounts: Eventual, - escrow_subgraph: &'static SubgraphClient, - escrow_adapter: EscrowAdapter, - tap_eip712_domain_separator: Eip712Domain, - sender_aggregator_endpoint: String, - sender_account_ref: ActorRef, - ) -> Self { - let required_checks: Vec> = vec![ - Arc::new(AllocationId::new( - sender, - allocation_id, - escrow_subgraph, - config, - )), - Arc::new(Signature::new( - tap_eip712_domain_separator.clone(), - escrow_accounts.clone(), - )), - ]; - let context = TapAgentContext::new( - pgpool.clone(), - allocation_id, - sender, - escrow_accounts.clone(), - escrow_adapter, - ); - let tap_manager = TapManager::new( - tap_eip712_domain_separator.clone(), - context, - Checks::new(required_checks), - ); - - Self { - pgpool, - tap_manager, - allocation_id, - sender, - sender_aggregator_endpoint, - config, - escrow_accounts, - tap_eip712_domain_separator, - sender_account_ref, - } - } - +impl SenderAllocationState { /// Delete obsolete receipts in the DB w.r.t. the last RAV in DB, then update the tap manager /// with the latest unaggregated fees from the database. async fn calculate_unaggregated_fee(&self) -> Result { @@ -396,7 +413,7 @@ impl SenderAllocation { let encoded_signature = receipt.signature.to_vec(); let receipt_signer = receipt - .recover_signer(&self.tap_eip712_domain_separator) + .recover_signer(&self.domain_separator) .map_err(|e| { error!("Failed to recover receipt signer: {}", e); anyhow!(e) @@ -537,22 +554,22 @@ mod tests { .await .unwrap(); - let allocation = SenderAllocation::new( + let args = SenderAllocationArgs { config, - pgpool.clone(), - *ALLOCATION_ID_0, - SENDER.1, - escrow_accounts_eventual, + pgpool: pgpool.clone(), + allocation_id: *ALLOCATION_ID_0, + sender: SENDER.1, + escrow_accounts: escrow_accounts_eventual, escrow_subgraph, escrow_adapter, - TAP_EIP712_DOMAIN_SEPARATOR.clone(), + domain_separator: TAP_EIP712_DOMAIN_SEPARATOR.clone(), sender_aggregator_endpoint, sender_account_ref, - ) - .await; + }; - let (allocation_ref, _join_handle) = - SenderAllocation::spawn(None, allocation, ()).await.unwrap(); + let (allocation_ref, _join_handle) = SenderAllocation::spawn(None, SenderAllocation, args) + .await + .unwrap(); allocation_ref } From 857ddadb0883d1c8ec6ab10ac88da3c6107c39df Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Mon, 1 Apr 2024 20:45:32 -0300 Subject: [PATCH 15/41] chore: add license to files Signed-off-by: Gustavo Inacio --- tap-agent/src/agent/allocation_id_tracker.rs | 3 +++ tap-agent/tests/sender_allocation_tests.rs | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tap-agent/src/agent/allocation_id_tracker.rs b/tap-agent/src/agent/allocation_id_tracker.rs index 8ee702329..6b8e55129 100644 --- a/tap-agent/src/agent/allocation_id_tracker.rs +++ b/tap-agent/src/agent/allocation_id_tracker.rs @@ -1,3 +1,6 @@ +// Copyright 2023-, GraphOps and Semiotic Labs. +// SPDX-License-Identifier: Apache-2.0 + use alloy_primitives::Address; use std::collections::{BTreeMap, HashMap}; diff --git a/tap-agent/tests/sender_allocation_tests.rs b/tap-agent/tests/sender_allocation_tests.rs index 8b1378917..cf206ebfd 100644 --- a/tap-agent/tests/sender_allocation_tests.rs +++ b/tap-agent/tests/sender_allocation_tests.rs @@ -1 +1,2 @@ - +// Copyright 2023-, GraphOps and Semiotic Labs. +// SPDX-License-Identifier: Apache-2.0 From 0bcc0ef10d9720bfdec813ffbaa47a4b52162745 Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Mon, 1 Apr 2024 21:47:15 -0300 Subject: [PATCH 16/41] test: add allocation_id_tracker unit test Signed-off-by: Gustavo Inacio --- tap-agent/src/agent/allocation_id_tracker.rs | 59 +++++++++++++++++++- 1 file changed, 56 insertions(+), 3 deletions(-) diff --git a/tap-agent/src/agent/allocation_id_tracker.rs b/tap-agent/src/agent/allocation_id_tracker.rs index 6b8e55129..adf916308 100644 --- a/tap-agent/src/agent/allocation_id_tracker.rs +++ b/tap-agent/src/agent/allocation_id_tracker.rs @@ -29,9 +29,13 @@ impl AllocationIdTracker { } } - self.id_to_fee.insert(id, fee); - self.total_fee += fee; - *self.fee_to_count.entry(fee).or_insert(0) += 1; + if fee > 0 { + self.id_to_fee.insert(id, fee); + self.total_fee += fee; + *self.fee_to_count.entry(fee).or_insert(0) += 1; + } else { + self.id_to_fee.remove(&id); + } } pub fn get_heaviest_allocation_id(&self) -> Option
{ @@ -47,3 +51,52 @@ impl AllocationIdTracker { self.total_fee } } + +#[cfg(test)] +mod tests { + use crate::tap::test_utils::{ALLOCATION_ID_0, ALLOCATION_ID_1, ALLOCATION_ID_2}; + + use super::AllocationIdTracker; + #[test] + fn test_allocation_id_tracker() { + let allocation_id_0 = *ALLOCATION_ID_0; + let allocation_id_1 = *ALLOCATION_ID_1; + let allocation_id_2 = *ALLOCATION_ID_2; + + let mut tracker = AllocationIdTracker::new(); + assert_eq!(tracker.get_heaviest_allocation_id(), None); + assert_eq!(tracker.get_total_fee(), 0); + + tracker.add_or_update(allocation_id_0, 10); + assert_eq!(tracker.get_heaviest_allocation_id(), Some(allocation_id_0)); + assert_eq!(tracker.get_total_fee(), 10); + + tracker.add_or_update(allocation_id_2, 20); + assert_eq!(tracker.get_heaviest_allocation_id(), Some(allocation_id_2)); + assert_eq!(tracker.get_total_fee(), 30); + + tracker.add_or_update(allocation_id_1, 30); + assert_eq!(tracker.get_heaviest_allocation_id(), Some(allocation_id_1)); + assert_eq!(tracker.get_total_fee(), 60); + + tracker.add_or_update(allocation_id_2, 10); + assert_eq!(tracker.get_heaviest_allocation_id(), Some(allocation_id_1)); + assert_eq!(tracker.get_total_fee(), 50); + + tracker.add_or_update(allocation_id_2, 40); + assert_eq!(tracker.get_heaviest_allocation_id(), Some(allocation_id_2)); + assert_eq!(tracker.get_total_fee(), 80); + + tracker.add_or_update(allocation_id_1, 0); + assert_eq!(tracker.get_heaviest_allocation_id(), Some(allocation_id_2)); + assert_eq!(tracker.get_total_fee(), 50); + + tracker.add_or_update(allocation_id_2, 0); + assert_eq!(tracker.get_heaviest_allocation_id(), Some(allocation_id_0)); + assert_eq!(tracker.get_total_fee(), 10); + + tracker.add_or_update(allocation_id_0, 0); + assert_eq!(tracker.get_heaviest_allocation_id(), None); + assert_eq!(tracker.get_total_fee(), 0); + } +} From 2bf706e9fa16f8ead49c45b9907436cedde46571 Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Tue, 2 Apr 2024 13:59:24 -0300 Subject: [PATCH 17/41] test: move tests to integration tests Signed-off-by: Gustavo Inacio --- tap-agent/src/agent.rs | 8 +- tap-agent/src/agent/allocation_id_tracker.rs | 28 +- tap-agent/src/agent/sender_account.rs | 551 +----------------- .../src/agent/sender_accounts_manager.rs | 149 +---- tap-agent/src/agent/sender_allocation.rs | 245 ++------ tap-agent/src/lib.rs | 20 + tap-agent/src/main.rs | 20 +- tap-agent/src/tap/context/rav.rs | 29 +- tap-agent/src/tap/context/receipt.rs | 174 +++--- tap-agent/src/tap/mod.rs | 2 +- tap-agent/src/tap/test_utils.rs | 106 ---- tap-agent/tests/sender_account_tests.rs | 551 ++++++++++++++++++ tap-agent/tests/sender_allocation_tests.rs | 236 ++++++++ tap-agent/tests/sender_manager_tests.rs | 152 +++++ tap-agent/tests/test_utils.rs | 140 +++++ 15 files changed, 1309 insertions(+), 1102 deletions(-) create mode 100644 tap-agent/src/lib.rs create mode 100644 tap-agent/tests/sender_account_tests.rs create mode 100644 tap-agent/tests/sender_manager_tests.rs create mode 100644 tap-agent/tests/test_utils.rs diff --git a/tap-agent/src/agent.rs b/tap-agent/src/agent.rs index ff4f8c15f..5956749eb 100644 --- a/tap-agent/src/agent.rs +++ b/tap-agent/src/agent.rs @@ -15,11 +15,11 @@ use crate::config::{Cli, EscrowSubgraph, Ethereum, IndexerInfrastructure, Networ use crate::{aggregator_endpoints, database, CONFIG, EIP_712_DOMAIN}; use sender_accounts_manager::SenderAccountsManager; -mod allocation_id_tracker; -mod sender_account; +pub mod allocation_id_tracker; +pub mod sender_account; pub mod sender_accounts_manager; -mod sender_allocation; -mod unaggregated_receipts; +pub mod sender_allocation; +pub mod unaggregated_receipts; pub async fn start_agent() -> ActorRef { let Cli { diff --git a/tap-agent/src/agent/allocation_id_tracker.rs b/tap-agent/src/agent/allocation_id_tracker.rs index adf916308..74515a602 100644 --- a/tap-agent/src/agent/allocation_id_tracker.rs +++ b/tap-agent/src/agent/allocation_id_tracker.rs @@ -4,7 +4,7 @@ use alloy_primitives::Address; use std::collections::{BTreeMap, HashMap}; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] pub struct AllocationIdTracker { id_to_fee: HashMap, fee_to_count: BTreeMap, @@ -12,14 +12,6 @@ pub struct AllocationIdTracker { } impl AllocationIdTracker { - pub fn new() -> Self { - AllocationIdTracker { - id_to_fee: HashMap::new(), - fee_to_count: BTreeMap::new(), - total_fee: 0, - } - } - pub fn add_or_update(&mut self, id: Address, fee: u128) { if let Some(&old_fee) = self.id_to_fee.get(&id) { self.total_fee -= old_fee; @@ -54,16 +46,20 @@ impl AllocationIdTracker { #[cfg(test)] mod tests { - use crate::tap::test_utils::{ALLOCATION_ID_0, ALLOCATION_ID_1, ALLOCATION_ID_2}; - use super::AllocationIdTracker; + use std::str::FromStr; + use thegraph::types::Address; + #[test] fn test_allocation_id_tracker() { - let allocation_id_0 = *ALLOCATION_ID_0; - let allocation_id_1 = *ALLOCATION_ID_1; - let allocation_id_2 = *ALLOCATION_ID_2; - - let mut tracker = AllocationIdTracker::new(); + let allocation_id_0: Address = + Address::from_str("0xabababababababababababababababababababab").unwrap(); + let allocation_id_1: Address = + Address::from_str("0xbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc").unwrap(); + let allocation_id_2: Address = + Address::from_str("0xcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd").unwrap(); + + let mut tracker = AllocationIdTracker::default(); assert_eq!(tracker.get_heaviest_allocation_id(), None); assert_eq!(tracker.get_total_fee(), 0); diff --git a/tap-agent/src/agent/sender_account.rs b/tap-agent/src/agent/sender_account.rs index 50e9e94f7..7f13d0156 100644 --- a/tap-agent/src/agent/sender_account.rs +++ b/tap-agent/src/agent/sender_account.rs @@ -26,7 +26,6 @@ pub enum SenderAccountMessage { UpdateAllocationIds(HashSet
), RemoveSenderAccount, UpdateReceiptFees(Address, UnaggregatedReceipts), - #[cfg(test)] GetAllocationTracker(ractor::RpcReplyPort), } @@ -147,7 +146,7 @@ impl Actor for SenderAccount { let escrow_adapter = EscrowAdapter::new(escrow_accounts.clone(), sender_id); Ok(State { - allocation_id_tracker: AllocationIdTracker::new(), + allocation_id_tracker: AllocationIdTracker::default(), allocation_ids, _indexer_allocations_handle, prefix, @@ -217,7 +216,7 @@ impl Actor for SenderAccount { state.allocation_ids = allocation_ids; } - #[cfg(test)] + // #[cfg(test)] SenderAccountMessage::GetAllocationTracker(reply) => { if !reply.is_closed() { let _ = reply.send(state.allocation_id_tracker.clone()); @@ -230,544 +229,22 @@ impl Actor for SenderAccount { #[cfg(test)] mod tests { - use crate::agent::sender_accounts_manager::NewReceiptNotification; - use alloy_primitives::hex::ToHex; - use bigdecimal::{num_bigint::ToBigInt, ToPrimitive}; - use indexer_common::subgraph_client::DeploymentDetails; - use ractor::cast; - use serde_json::json; - use std::str::FromStr; - use std::time::Duration; - use std::{collections::HashMap, sync::atomic::AtomicU32}; - use tap_aggregator::server::run_server; - use tap_core::{rav::ReceiptAggregateVoucher, signed_message::EIP712SignedMessage}; - use tokio::task::JoinHandle; - use wiremock::{ - matchers::{body_string_contains, method}, - Mock, MockServer, ResponseTemplate, - }; - use crate::tap::test_utils::{ - create_received_receipt, store_receipt, ALLOCATION_ID_0, ALLOCATION_ID_1, ALLOCATION_ID_2, - INDEXER, SENDER, SIGNER, TAP_EIP712_DOMAIN_SEPARATOR, - }; + #[test] + fn test_update_allocation_ids() {} - use super::*; + #[test] + fn test_update_receipt_fees_no_rav() {} - const DUMMY_URL: &str = "http://localhost:1234"; - const VALUE_PER_RECEIPT: u64 = 100; - const TRIGGER_VALUE: u64 = 500; - static PREFIX_ID: AtomicU32 = AtomicU32::new(0); + #[test] + fn test_update_receipt_fees_trigger_rav() {} - // To help with testing from other modules. + #[test] + fn test_remove_sender_account() {} - async fn create_sender_with_allocations( - pgpool: PgPool, - sender_aggregator_endpoint: String, - escrow_subgraph_endpoint: &str, - ) -> (ActorRef, JoinHandle<()>, String) { - let config = Box::leak(Box::new(config::Cli { - config: None, - ethereum: config::Ethereum { - indexer_address: INDEXER.1, - }, - tap: config::Tap { - rav_request_trigger_value: TRIGGER_VALUE, - rav_request_timestamp_buffer_ms: 1, - rav_request_timeout_secs: 5, - ..Default::default() - }, - ..Default::default() - })); + #[test] + fn test_create_sender_allocation() {} - let escrow_subgraph = Box::leak(Box::new(SubgraphClient::new( - reqwest::Client::new(), - None, - DeploymentDetails::for_query_url(escrow_subgraph_endpoint).unwrap(), - ))); - - let escrow_accounts_eventual = Eventual::from_value(EscrowAccounts::new( - HashMap::from([(SENDER.1, 1000.into())]), - HashMap::from([(SENDER.1, vec![SIGNER.1])]), - )); - - let indexer_allocations = Eventual::from_value(HashSet::from([ - *ALLOCATION_ID_0, - *ALLOCATION_ID_1, - *ALLOCATION_ID_2, - ])); - - let prefix = format!( - "test-{}", - PREFIX_ID.fetch_add(1, std::sync::atomic::Ordering::SeqCst) - ); - - let args = SenderAccountArgs { - config, - pgpool, - sender_id: SENDER.1, - escrow_accounts: escrow_accounts_eventual, - indexer_allocations, - escrow_subgraph, - domain_separator: TAP_EIP712_DOMAIN_SEPARATOR.clone(), - sender_aggregator_endpoint, - allocation_ids: HashSet::new(), - prefix: Some(prefix.clone()), - }; - - let (sender, handle) = SenderAccount::spawn(Some(prefix.clone()), SenderAccount, args) - .await - .unwrap(); - - // await for the allocations to be created - ractor::concurrency::sleep(Duration::from_millis(100)).await; - - (sender, handle, prefix) - } - - /// Test that the sender_account correctly ignores new receipt notifications with - /// an ID lower than the last receipt ID processed (be it from the DB or from a prior receipt - /// notification). - #[sqlx::test(migrations = "../migrations")] - async fn test_handle_new_receipt_notification(pgpool: PgPool) { - // Add receipts to the database. Before creating the sender and allocation so that it loads - // the receipts from the DB. - let mut expected_unaggregated_fees = 0u128; - for i in 10..20 { - let receipt = - create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i, i.into()).await; - store_receipt(&pgpool, receipt.signed_receipt()) - .await - .unwrap(); - expected_unaggregated_fees += u128::from(i); - } - - let (sender, handle, prefix) = - create_sender_with_allocations(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL).await; - - let allocation = ActorRef::::where_is(format!( - "{prefix}:{sender}:{allocation_id}", - sender = SENDER.1, - allocation_id = *ALLOCATION_ID_0 - )) - .unwrap(); - - // Check that the sender's unaggregated fees are correct. - let allocation_tracker = call!(sender, SenderAccountMessage::GetAllocationTracker).unwrap(); - assert_eq!( - allocation_tracker.get_total_fee(), - expected_unaggregated_fees - ); - - let allocation_unaggregated_fees = - call!(allocation, SenderAllocationMessage::GetUnaggregatedReceipts).unwrap(); - - // Check that the allocation's unaggregated fees are correct. - assert_eq!( - allocation_unaggregated_fees.value, - expected_unaggregated_fees - ); - - // Send a new receipt notification that has a lower ID than the last loaded from the DB. - // The last ID in the DB should be 10, since we added 10 receipts to the empty receipts - // table - let new_receipt_notification = NewReceiptNotification { - allocation_id: *ALLOCATION_ID_0, - signer_address: SIGNER.1, - id: 10, - timestamp_ns: 19, - value: 19, - }; - allocation - .cast(SenderAllocationMessage::NewReceipt( - new_receipt_notification, - )) - .unwrap(); - ractor::concurrency::sleep(Duration::from_millis(10)).await; - - let allocation_unaggregated_fees = - call!(allocation, SenderAllocationMessage::GetUnaggregatedReceipts).unwrap(); - - // Check that the allocation's unaggregated fees have *not* increased. - assert_eq!( - allocation_unaggregated_fees.value, - expected_unaggregated_fees - ); - - // Check that the unaggregated fees have *not* increased. - let allocation_tracker = call!(sender, SenderAccountMessage::GetAllocationTracker).unwrap(); - assert_eq!( - allocation_tracker.get_total_fee(), - expected_unaggregated_fees - ); - - // Send a new receipt notification. - let new_receipt_notification = NewReceiptNotification { - allocation_id: *ALLOCATION_ID_0, - signer_address: SIGNER.1, - id: 30, - timestamp_ns: 20, - value: 20, - }; - allocation - .cast(SenderAllocationMessage::NewReceipt( - new_receipt_notification, - )) - .unwrap(); - ractor::concurrency::sleep(Duration::from_millis(10)).await; - - expected_unaggregated_fees += 20; - - let allocation_unaggregated_fees = - call!(allocation, SenderAllocationMessage::GetUnaggregatedReceipts).unwrap(); - - // Check that the allocation's unaggregated fees are correct. - assert_eq!( - allocation_unaggregated_fees.value, - expected_unaggregated_fees - ); - - // Check that the sender's unaggregated fees are correct. - let allocation_tracker = call!(sender, SenderAccountMessage::GetAllocationTracker).unwrap(); - assert_eq!( - allocation_tracker.get_total_fee(), - expected_unaggregated_fees - ); - - // Send a new receipt notification that has a lower ID than the previous one. - let new_receipt_notification = NewReceiptNotification { - allocation_id: *ALLOCATION_ID_0, - signer_address: SIGNER.1, - id: 25, - timestamp_ns: 19, - value: 19, - }; - allocation - .cast(SenderAllocationMessage::NewReceipt( - new_receipt_notification, - )) - .unwrap(); - - ractor::concurrency::sleep(Duration::from_millis(10)).await; - - let allocation_unaggregated_fees = - call!(allocation, SenderAllocationMessage::GetUnaggregatedReceipts).unwrap(); - - // Check that the allocation's unaggregated fees have *not* increased. - assert_eq!( - allocation_unaggregated_fees.value, - expected_unaggregated_fees - ); - - // Check that the unaggregated fees have *not* increased. - let allocation_tracker = call!(sender, SenderAccountMessage::GetAllocationTracker).unwrap(); - assert_eq!( - allocation_tracker.get_total_fee(), - expected_unaggregated_fees - ); - - sender.stop(None); - handle.await.unwrap(); - } - - #[sqlx::test(migrations = "../migrations")] - async fn test_rav_requester_auto(pgpool: PgPool) { - // Start a TAP aggregator server. - let (handle, aggregator_endpoint) = run_server( - 0, - SIGNER.0.clone(), - vec![SIGNER.1].into_iter().collect(), - TAP_EIP712_DOMAIN_SEPARATOR.clone(), - 100 * 1024, - 100 * 1024, - 1, - ) - .await - .unwrap(); - - // Start a mock graphql server using wiremock - let mock_server = MockServer::start().await; - - // Mock result for TAP redeem txs for (allocation, sender) pair. - mock_server - .register( - Mock::given(method("POST")) - .and(body_string_contains("transactions")) - .respond_with( - ResponseTemplate::new(200) - .set_body_json(json!({ "data": { "transactions": []}})), - ), - ) - .await; - - // Create a sender_account. - let (sender_account, sender_handle, prefix) = create_sender_with_allocations( - pgpool.clone(), - "http://".to_owned() + &aggregator_endpoint.to_string(), - &mock_server.uri(), - ) - .await; - - let allocation = ActorRef::::where_is(format!( - "{prefix}:{sender}:{allocation_id}", - sender = SENDER.1, - allocation_id = *ALLOCATION_ID_0 - )) - .unwrap(); - - // Add receipts to the database and call the `handle_new_receipt_notification` method - // correspondingly. - let mut total_value = 0; - let mut trigger_value = 0; - for i in 1..=10 { - // These values should be enough to trigger a RAV request at i == 7 since we set the - // `rav_request_trigger_value` to 100. - let value = (i + VALUE_PER_RECEIPT) as u128; - - let receipt = - create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i + 1, value).await; - store_receipt(&pgpool, receipt.signed_receipt()) - .await - .unwrap(); - let new_receipt_notification = NewReceiptNotification { - allocation_id: *ALLOCATION_ID_0, - signer_address: SIGNER.1, - id: i, - timestamp_ns: i + 1, - value, - }; - allocation - .cast(SenderAllocationMessage::NewReceipt( - new_receipt_notification, - )) - .unwrap(); - - ractor::concurrency::sleep(Duration::from_millis(100)).await; - - total_value += value; - if total_value >= TRIGGER_VALUE as u128 && trigger_value == 0 { - trigger_value = total_value; - } - } - - ractor::concurrency::sleep(Duration::from_millis(10)).await; - - // Wait for the RAV requester to finish. - for _ in 0..100 { - let allocation_tracker = - call!(sender_account, SenderAccountMessage::GetAllocationTracker).unwrap(); - if allocation_tracker.get_total_fee() < trigger_value { - break; - } - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - } - - // Get the latest RAV from the database. - let latest_rav = sqlx::query!( - r#" - SELECT signature, allocation_id, timestamp_ns, value_aggregate - FROM scalar_tap_ravs - WHERE allocation_id = $1 AND sender_address = $2 - "#, - ALLOCATION_ID_0.encode_hex::(), - SENDER.1.encode_hex::() - ) - .fetch_optional(&pgpool) - .await - .unwrap() - .unwrap(); - - let latest_rav = EIP712SignedMessage { - message: ReceiptAggregateVoucher { - allocationId: Address::from_str(&latest_rav.allocation_id).unwrap(), - timestampNs: latest_rav.timestamp_ns.to_u64().unwrap(), - // Beware, BigDecimal::to_u128() actually uses to_u64() under the hood... - // So we're converting to BigInt to get a proper implementation of to_u128(). - valueAggregate: latest_rav - .value_aggregate - .to_bigint() - .map(|v| v.to_u128()) - .unwrap() - .unwrap(), - }, - signature: latest_rav.signature.as_slice().try_into().unwrap(), - }; - - // Check that the latest RAV value is correct. - assert!(latest_rav.message.valueAggregate >= trigger_value); - - // Check that the allocation's unaggregated fees value is reduced. - - let allocation_unaggregated_fees = - call!(allocation, SenderAllocationMessage::GetUnaggregatedReceipts).unwrap(); - - // Check that the allocation's unaggregated fees have *not* increased. - assert!(allocation_unaggregated_fees.value <= trigger_value); - - // Check that the sender's unaggregated fees value is reduced. - let allocation_tracker = - call!(sender_account, SenderAccountMessage::GetAllocationTracker).unwrap(); - assert!(allocation_tracker.get_total_fee() <= trigger_value); - - // Reset the total value and trigger value. - total_value = allocation_tracker.get_total_fee(); - trigger_value = 0; - - // Add more receipts - for i in 10..20 { - let value = (i + VALUE_PER_RECEIPT) as u128; - - let receipt = - create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i + 1, i.into()).await; - store_receipt(&pgpool, receipt.signed_receipt()) - .await - .unwrap(); - - let new_receipt_notification = NewReceiptNotification { - allocation_id: *ALLOCATION_ID_0, - signer_address: SIGNER.1, - id: i, - timestamp_ns: i + 1, - value, - }; - allocation - .cast(SenderAllocationMessage::NewReceipt( - new_receipt_notification, - )) - .unwrap(); - - ractor::concurrency::sleep(Duration::from_millis(10)).await; - - total_value += value; - if total_value >= TRIGGER_VALUE as u128 && trigger_value == 0 { - trigger_value = total_value; - } - } - - // Wait for the RAV requester to finish. - for _ in 0..100 { - let allocation_tracker = - call!(sender_account, SenderAccountMessage::GetAllocationTracker).unwrap(); - if allocation_tracker.get_total_fee() < trigger_value { - break; - } - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - } - - // Get the latest RAV from the database. - let latest_rav = sqlx::query!( - r#" - SELECT signature, allocation_id, timestamp_ns, value_aggregate - FROM scalar_tap_ravs - WHERE allocation_id = $1 AND sender_address = $2 - "#, - ALLOCATION_ID_0.encode_hex::(), - SENDER.1.encode_hex::() - ) - .fetch_optional(&pgpool) - .await - .unwrap() - .unwrap(); - - let latest_rav = EIP712SignedMessage { - message: ReceiptAggregateVoucher { - allocationId: Address::from_str(&latest_rav.allocation_id).unwrap(), - timestampNs: latest_rav.timestamp_ns.to_u64().unwrap(), - // Beware, BigDecimal::to_u128() actually uses to_u64() under the hood... - // So we're converting to BigInt to get a proper implementation of to_u128(). - valueAggregate: latest_rav - .value_aggregate - .to_bigint() - .map(|v| v.to_u128()) - .unwrap() - .unwrap(), - }, - signature: latest_rav.signature.as_slice().try_into().unwrap(), - }; - - // Check that the latest RAV value is correct. - - assert!(latest_rav.message.valueAggregate >= trigger_value); - - // Check that the allocation's unaggregated fees value is reduced. - - let allocation_unaggregated_fees = - call!(allocation, SenderAllocationMessage::GetUnaggregatedReceipts).unwrap(); - - // Check that the allocation's unaggregated fees have *not* increased. - assert!(allocation_unaggregated_fees.value <= trigger_value); - - // Check that the unaggregated fees value is reduced. - let allocation_tracker = - call!(sender_account, SenderAccountMessage::GetAllocationTracker).unwrap(); - assert!(allocation_tracker.get_total_fee() <= trigger_value); - - // Stop the TAP aggregator server. - handle.stop().unwrap(); - handle.stopped().await; - - sender_account.stop(None); - sender_handle.await.unwrap(); - } - - #[sqlx::test(migrations = "../migrations")] - async fn test_sender_unaggregated_fees(pgpool: PgPool) { - // Create a sender_account. - let (sender_account, handle, _prefix) = - create_sender_with_allocations(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL).await; - - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - - // Closure that adds a number of receipts to an allocation. - - let update_receipt_fees = |sender_account: &ActorRef, - allocation_id: Address, - value: u128| { - cast!( - sender_account, - SenderAccountMessage::UpdateReceiptFees( - allocation_id, - UnaggregatedReceipts { - value, - ..Default::default() - }, - ) - ) - .unwrap(); - - value - }; - - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - - // Add receipts to the database for allocation_0 - let total_value_0 = update_receipt_fees(&sender_account, *ALLOCATION_ID_0, 90); - tokio::time::sleep(std::time::Duration::from_millis(10)).await; - - // Add receipts to the database for allocation_1 - let total_value_1 = update_receipt_fees(&sender_account, *ALLOCATION_ID_1, 100); - tokio::time::sleep(std::time::Duration::from_millis(10)).await; - - // Add receipts to the database for allocation_2 - let total_value_2 = update_receipt_fees(&sender_account, *ALLOCATION_ID_2, 80); - tokio::time::sleep(std::time::Duration::from_millis(10)).await; - - let allocation_tracker = - call!(sender_account, SenderAccountMessage::GetAllocationTracker).unwrap(); - - // Get the heaviest allocation. - let heaviest_allocation = allocation_tracker.get_heaviest_allocation_id().unwrap(); - - // Check that the heaviest allocation is correct. - assert_eq!(heaviest_allocation, *ALLOCATION_ID_1); - - // Check that the sender's unaggregated fees value is correct. - assert_eq!( - allocation_tracker.get_total_fee(), - total_value_0 + total_value_1 + total_value_2 - ); - - sender_account.stop(None); - handle.await.unwrap(); - } + #[test] + fn test_create_sender_account() {} } diff --git a/tap-agent/src/agent/sender_accounts_manager.rs b/tap-agent/src/agent/sender_accounts_manager.rs index 03892c7ff..ec992c10a 100644 --- a/tap-agent/src/agent/sender_accounts_manager.rs +++ b/tap-agent/src/agent/sender_accounts_manager.rs @@ -374,144 +374,15 @@ async fn new_receipts_watcher( #[cfg(test)] mod tests { + #[test] + fn test_create_sender_accounts_manager() {} - use std::vec; - - use ethereum_types::U256; - use indexer_common::{ - prelude::{AllocationStatus, SubgraphDeployment}, - subgraph_client::DeploymentDetails, - }; - use ractor::ActorStatus; - use serde_json::json; - use thegraph::types::DeploymentId; - use wiremock::{ - matchers::{body_string_contains, method}, - Mock, MockServer, ResponseTemplate, - }; - - use crate::tap::test_utils::{INDEXER, SENDER, SIGNER, TAP_EIP712_DOMAIN_SEPARATOR}; - - use super::*; - - #[sqlx::test(migrations = "../migrations")] - async fn test_sender_account_creation_and_eol(pgpool: PgPool) { - let config = Box::leak(Box::new(config::Cli { - config: None, - ethereum: config::Ethereum { - indexer_address: INDEXER.1, - }, - tap: config::Tap { - rav_request_trigger_value: 100, - rav_request_timestamp_buffer_ms: 1, - ..Default::default() - }, - ..Default::default() - })); - - let (mut indexer_allocations_writer, indexer_allocations_eventual) = - Eventual::>::new(); - indexer_allocations_writer.write(HashMap::new()); - - let (mut escrow_accounts_writer, escrow_accounts_eventual) = - Eventual::::new(); - escrow_accounts_writer.write(EscrowAccounts::default()); - - // Mock escrow subgraph. - let mock_server = MockServer::start().await; - mock_server - .register( - Mock::given(method("POST")) - .and(body_string_contains("transactions")) - .respond_with( - ResponseTemplate::new(200) - .set_body_json(json!({ "data": { "transactions": []}})), - ), - ) - .await; - let escrow_subgraph = Box::leak(Box::new(SubgraphClient::new( - reqwest::Client::new(), - None, - DeploymentDetails::for_query_url(&mock_server.uri()).unwrap(), - ))); - - let args = SenderAccountsManagerArgs { - config, - domain_separator: TAP_EIP712_DOMAIN_SEPARATOR.clone(), - pgpool: pgpool.clone(), - indexer_allocations: indexer_allocations_eventual, - escrow_accounts: escrow_accounts_eventual, - escrow_subgraph, - sender_aggregator_endpoints: HashMap::from([( - SENDER.1, - String::from("http://localhost:8000"), - )]), - }; - SenderAccountsManager::spawn(None, SenderAccountsManager, args) - .await - .unwrap(); - - let allocation_id = - Address::from_str("0xdd975e30aafebb143e54d215db8a3e8fd916a701").unwrap(); - - // Add an allocation to the indexer_allocations Eventual. - indexer_allocations_writer.write(HashMap::from([( - allocation_id, - Allocation { - id: allocation_id, - indexer: INDEXER.1, - allocated_tokens: U256::from_str("601726452999999979510903").unwrap(), - created_at_block_hash: - "0x99d3fbdc0105f7ccc0cd5bb287b82657fe92db4ea8fb58242dafb90b1c6e2adf".to_string(), - created_at_epoch: 953, - closed_at_epoch: None, - subgraph_deployment: SubgraphDeployment { - id: DeploymentId::from_str( - "0xcda7fa0405d6fd10721ed13d18823d24b535060d8ff661f862b26c23334f13bf" - ).unwrap(), - denied_at: Some(0), - }, - status: AllocationStatus::Null, - closed_at_epoch_start_block_hash: None, - previous_epoch_start_block_hash: None, - poi: None, - query_fee_rebates: None, - query_fees_collected: None, - }, - )])); - - // Add an escrow account to the escrow_accounts Eventual. - escrow_accounts_writer.write(EscrowAccounts::new( - HashMap::from([(SENDER.1, 1000.into())]), - HashMap::from([(SENDER.1, vec![SIGNER.1])]), - )); - - // Wait for the SenderAccount to be created. - tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; - - // Check that the SenderAccount was created. - let sender_account = - ActorRef::::where_is(SENDER.1.to_string()).unwrap(); - assert_eq!(sender_account.get_status(), ActorStatus::Running); - - let sender_allocation_id = format!("{}:{}", SENDER.1, allocation_id); - let sender_allocation = - ActorRef::::where_is(sender_allocation_id).unwrap(); - assert_eq!(sender_allocation.get_status(), ActorStatus::Running); - - // Remove the escrow account from the escrow_accounts Eventual. - escrow_accounts_writer.write(EscrowAccounts::default()); - - // Wait a bit - tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; - - // Check that the Sender's allocation moved from active to ineligible. - - assert_eq!(sender_account.get_status(), ActorStatus::Stopped); - assert_eq!(sender_allocation.get_status(), ActorStatus::Stopped); - - let sender_account = ActorRef::::where_is(SENDER.1.to_string()); - - assert!(sender_account.is_none()); - } + #[test] + fn test_create_with_pending_sender_allocations() {} + + #[test] + fn test_update_sender_allocation() {} + + #[test] + fn test_create_sender_account() {} } diff --git a/tap-agent/src/agent/sender_allocation.rs b/tap-agent/src/agent/sender_allocation.rs index 7b44cbd74..8f6c3c403 100644 --- a/tap-agent/src/agent/sender_allocation.rs +++ b/tap-agent/src/agent/sender_allocation.rs @@ -69,8 +69,6 @@ pub enum SenderAllocationMessage { NewReceipt(NewReceiptNotification), TriggerRAVRequest(RpcReplyPort), CloseAllocation, - - #[cfg(test)] GetUnaggregatedReceipts(RpcReplyPort), } @@ -220,8 +218,6 @@ impl Actor for SenderAllocation { })?; myself.stop(None); } - - #[cfg(test)] SenderAllocationMessage::GetUnaggregatedReceipts(reply) => { if !reply.is_closed() { let _ = reply.send(unaggreated_fees.clone()); @@ -480,228 +476,65 @@ impl SenderAllocationState { #[cfg(test)] mod tests { - use std::collections::HashMap; - - use indexer_common::subgraph_client::DeploymentDetails; - use ractor::call; - use serde_json::json; - use tap_aggregator::server::run_server; - - use wiremock::{ - matchers::{body_string_contains, method}, - Mock, MockServer, ResponseTemplate, - }; + #[test] + fn test_create_sender_allocation() { + // create an allocation - struct MockSenderAccount; + // expect nothing fails? + } - #[async_trait::async_trait] - impl Actor for MockSenderAccount { - type Msg = SenderAccountMessage; - type State = (); - type Arguments = (); + #[test] + fn test_receive_new_receipt() { + // should validate with id less than last_id - async fn pre_start( - &self, - _myself: ActorRef, - _allocation_ids: Self::Arguments, - ) -> std::result::Result { - Ok(()) - } + // should emit update aggregate fees message to sender account } - use super::*; - use crate::tap::test_utils::{ - create_rav, create_received_receipt, store_rav, store_receipt, ALLOCATION_ID_0, INDEXER, - SENDER, SIGNER, TAP_EIP712_DOMAIN_SEPARATOR, - }; - - const DUMMY_URL: &str = "http://localhost:1234"; - - async fn create_sender_allocation( - pgpool: PgPool, - sender_aggregator_endpoint: String, - escrow_subgraph_endpoint: &str, - ) -> ActorRef { - let config = Box::leak(Box::new(config::Cli { - config: None, - ethereum: config::Ethereum { - indexer_address: INDEXER.1, - }, - tap: config::Tap { - rav_request_trigger_value: 100, - rav_request_timestamp_buffer_ms: 1, - rav_request_timeout_secs: 5, - ..Default::default() - }, - ..Default::default() - })); - - let escrow_subgraph = Box::leak(Box::new(SubgraphClient::new( - reqwest::Client::new(), - None, - DeploymentDetails::for_query_url(escrow_subgraph_endpoint).unwrap(), - ))); - - let escrow_accounts_eventual = Eventual::from_value(EscrowAccounts::new( - HashMap::from([(SENDER.1, 1000.into())]), - HashMap::from([(SENDER.1, vec![SIGNER.1])]), - )); - - let escrow_adapter = EscrowAdapter::new(escrow_accounts_eventual.clone(), SENDER.1); - - let (sender_account_ref, _join_handle) = - MockSenderAccount::spawn(None, MockSenderAccount, ()) - .await - .unwrap(); - - let args = SenderAllocationArgs { - config, - pgpool: pgpool.clone(), - allocation_id: *ALLOCATION_ID_0, - sender: SENDER.1, - escrow_accounts: escrow_accounts_eventual, - escrow_subgraph, - escrow_adapter, - domain_separator: TAP_EIP712_DOMAIN_SEPARATOR.clone(), - sender_aggregator_endpoint, - sender_account_ref, - }; + #[test] + fn test_trigger_rav_request() { + // should rav request - let (allocation_ref, _join_handle) = SenderAllocation::spawn(None, SenderAllocation, args) - .await - .unwrap(); - - allocation_ref + // should return correct unaggregated fees } - /// Test that the sender_allocation correctly updates the unaggregated fees from the - /// database when there is no RAV in the database. - /// - /// The sender_allocation should consider all receipts found for the allocation and - /// sender. - #[sqlx::test(migrations = "../migrations")] - async fn test_update_unaggregated_fees_no_rav(pgpool: PgPool) { - // Add receipts to the database. - for i in 1..10 { - let receipt = - create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i, i.into()).await; - store_receipt(&pgpool, receipt.signed_receipt()) - .await - .unwrap(); - } - - let sender_allocation = - create_sender_allocation(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL).await; + #[test] + fn test_close_allocation() { + // create allocation - // Get total_unaggregated_fees - let total_unaggregated_fees = call!( - sender_allocation, - SenderAllocationMessage::GetUnaggregatedReceipts - ) - .unwrap(); + // populate db - // Check that the unaggregated fees are correct. - assert_eq!(total_unaggregated_fees.value, 45u128); - } + // should trigger rav request - /// Test that the sender_allocation correctly updates the unaggregated fees from the - /// database when there is a RAV in the database as well as receipts which timestamp are lesser - /// and greater than the RAV's timestamp. - /// - /// The sender_allocation should only consider receipts with a timestamp greater - /// than the RAV's timestamp. - #[sqlx::test(migrations = "../migrations")] - async fn test_update_unaggregated_fees_with_rav(pgpool: PgPool) { - // Add the RAV to the database. - // This RAV has timestamp 4. The sender_allocation should only consider receipts - // with a timestamp greater than 4. - let signed_rav = create_rav(*ALLOCATION_ID_0, SIGNER.0.clone(), 4, 10).await; - store_rav(&pgpool, signed_rav, SENDER.1).await.unwrap(); - - // Add receipts to the database. - for i in 1..10 { - let receipt = - create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i, i.into()).await; - store_receipt(&pgpool, receipt.signed_receipt()) - .await - .unwrap(); - } + // should mark rav final - let sender_allocation = - create_sender_allocation(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL).await; + // should stop actor - // Get total_unaggregated_fees - let total_unaggregated_fees = call!( - sender_allocation, - SenderAllocationMessage::GetUnaggregatedReceipts - ) - .unwrap(); + // check if message is sent to sender account + } - // Check that the unaggregated fees are correct. - assert_eq!(total_unaggregated_fees.value, 35u128); + #[test] + fn test_store_failed_rav() { + // check database if failed rav is stored } - #[sqlx::test(migrations = "../migrations")] - async fn test_rav_requester_manual(pgpool: PgPool) { - // Start a TAP aggregator server. - let (handle, aggregator_endpoint) = run_server( - 0, - SIGNER.0.clone(), - vec![SIGNER.1].into_iter().collect(), - TAP_EIP712_DOMAIN_SEPARATOR.clone(), - 100 * 1024, - 100 * 1024, - 1, - ) - .await - .unwrap(); - - // Start a mock graphql server using wiremock - let mock_server = MockServer::start().await; - - // Mock result for TAP redeem txs for (allocation, sender) pair. - mock_server - .register( - Mock::given(method("POST")) - .and(body_string_contains("transactions")) - .respond_with( - ResponseTemplate::new(200) - .set_body_json(json!({ "data": { "transactions": []}})), - ), - ) - .await; - - // Add receipts to the database. - for i in 0..10 { - let receipt = - create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i + 1, i.into()).await; - store_receipt(&pgpool, receipt.signed_receipt()) - .await - .unwrap(); - } + #[test] + fn test_calculate_unaggregated_fee() { + // add a bunch of receipts - // Create a sender_allocation. - let sender_allocation = create_sender_allocation( - pgpool.clone(), - "http://".to_owned() + &aggregator_endpoint.to_string(), - &mock_server.uri(), - ) - .await; + // calculate unaggregated fee - // Trigger a RAV request manually. + // check if it's correct + } - // Get total_unaggregated_fees - let total_unaggregated_fees = call!( - sender_allocation, - SenderAllocationMessage::TriggerRAVRequest - ) - .unwrap(); + #[test] + fn test_store_invalid_receipts() { + // verify if invalid receipts are stored correctly + } - // Check that the unaggregated fees are correct. - assert_eq!(total_unaggregated_fees.value, 0u128); + #[test] + fn test_mark_rav_final() { + // mark rav as final - // Stop the TAP aggregator server. - handle.stop().unwrap(); - handle.stopped().await; + // check if database contains final rav } } diff --git a/tap-agent/src/lib.rs b/tap-agent/src/lib.rs new file mode 100644 index 000000000..5782273ae --- /dev/null +++ b/tap-agent/src/lib.rs @@ -0,0 +1,20 @@ +use alloy_sol_types::{eip712_domain, Eip712Domain}; +use lazy_static::lazy_static; + +use crate::config::Cli; + +lazy_static! { + pub static ref CONFIG: Cli = Cli::args(); + pub static ref EIP_712_DOMAIN: Eip712Domain = eip712_domain! { + name: "TAP", + version: "1", + chain_id: CONFIG.receipts.receipts_verifier_chain_id, + verifying_contract: CONFIG.receipts.receipts_verifier_address, + }; +} + +pub mod agent; +pub mod aggregator_endpoints; +pub mod config; +pub mod database; +pub mod tap; diff --git a/tap-agent/src/main.rs b/tap-agent/src/main.rs index 4d5b2c59f..ab2cc7cfd 100644 --- a/tap-agent/src/main.rs +++ b/tap-agent/src/main.rs @@ -1,29 +1,11 @@ // Copyright 2023-, GraphOps and Semiotic Labs. // SPDX-License-Identifier: Apache-2.0 -use alloy_sol_types::{eip712_domain, Eip712Domain}; use anyhow::Result; -use lazy_static::lazy_static; use tokio::signal::unix::{signal, SignalKind}; use tracing::{debug, info}; -use crate::config::Cli; - -mod agent; -mod aggregator_endpoints; -mod config; -mod database; -mod tap; - -lazy_static! { - pub static ref CONFIG: Cli = Cli::args(); - pub static ref EIP_712_DOMAIN: Eip712Domain = eip712_domain! { - name: "TAP", - version: "1", - chain_id: CONFIG.receipts.receipts_verifier_chain_id, - verifying_contract: CONFIG.receipts.receipts_verifier_address, - }; -} +use indexer_tap_agent::{agent, CONFIG}; #[tokio::main] async fn main() -> Result<()> { diff --git a/tap-agent/src/tap/context/rav.rs b/tap-agent/src/tap/context/rav.rs index 8979b75c2..dabccb026 100644 --- a/tap-agent/src/tap/context/rav.rs +++ b/tap-agent/src/tap/context/rav.rs @@ -127,15 +127,36 @@ impl RAVStore for TapAgentContext { #[cfg(test)] mod test { + use ethers_signers::LocalWallet; use eventuals::Eventual; use sqlx::PgPool; + use tap_core::signed_message::EIP712SignedMessage; use super::*; use crate::tap::{ escrow_adapter::EscrowAdapter, - test_utils::{create_rav, ALLOCATION_ID_0, SENDER, SIGNER}, + test_utils::{ALLOCATION_ID_0, SENDER, SIGNER, TAP_EIP712_DOMAIN_SEPARATOR}, }; + /// Fixture to generate a RAV using the wallet from `keys()` + fn create_rav( + allocation_id: Address, + signer_wallet: LocalWallet, + timestamp_ns: u64, + value_aggregate: u128, + ) -> SignedRAV { + EIP712SignedMessage::new( + &TAP_EIP712_DOMAIN_SEPARATOR, + ReceiptAggregateVoucher { + allocationId: allocation_id, + timestampNs: timestamp_ns, + valueAggregate: value_aggregate, + }, + &signer_wallet, + ) + .unwrap() + } + #[sqlx::test(migrations = "../migrations")] async fn update_and_retrieve_rav(pool: PgPool) { let timestamp_ns = u64::MAX - 10; @@ -154,8 +175,7 @@ mod test { SIGNER.0.clone(), timestamp_ns, value_aggregate, - ) - .await; + ); context.update_last_rav(new_rav.clone()).await.unwrap(); // Should trigger a retrieve_last_rav So eventually the last rav should be the one @@ -170,8 +190,7 @@ mod test { SIGNER.0.clone(), timestamp_ns + i, value_aggregate - (i as u128), - ) - .await; + ); context.update_last_rav(new_rav.clone()).await.unwrap(); } diff --git a/tap-agent/src/tap/context/receipt.rs b/tap-agent/src/tap/context/receipt.rs index cbfc75a6c..454246093 100644 --- a/tap-agent/src/tap/context/receipt.rs +++ b/tap-agent/src/tap/context/receipt.rs @@ -184,21 +184,76 @@ impl ReceiptDelete for TapAgentContext { #[cfg(test)] mod test { - use std::collections::HashMap; - use super::*; use crate::tap::{ escrow_adapter::EscrowAdapter, - test_utils::{ - create_received_receipt, store_receipt, ALLOCATION_ID_0, ALLOCATION_ID_IRRELEVANT, - SENDER, SENDER_IRRELEVANT, SIGNER, TAP_EIP712_DOMAIN_SEPARATOR, - }, + test_utils::{wallet, ALLOCATION_ID_0, SENDER, SIGNER, TAP_EIP712_DOMAIN_SEPARATOR}, }; use anyhow::Result; - + use bigdecimal::num_bigint::BigInt; + use ethers_signers::LocalWallet; use eventuals::Eventual; use indexer_common::escrow_accounts::EscrowAccounts; + use lazy_static::lazy_static; use sqlx::PgPool; + use std::collections::HashMap; + use tap_core::signed_message::EIP712SignedMessage; + + lazy_static! { + pub static ref SENDER_IRRELEVANT: (LocalWallet, Address) = wallet(1); + pub static ref ALLOCATION_ID_IRRELEVANT: Address = + Address::from_str("0xbcdebcdebcdebcdebcdebcdebcdebcdebcdebcde").unwrap(); + } + + /// Fixture to generate a signed receipt using the wallet from `keys()` and the + /// given `query_id` and `value` + fn create_received_receipt( + allocation_id: &Address, + signer_wallet: &LocalWallet, + nonce: u64, + timestamp_ns: u64, + value: u128, + ) -> ReceiptWithState { + let receipt = EIP712SignedMessage::new( + &TAP_EIP712_DOMAIN_SEPARATOR, + Receipt { + allocation_id: *allocation_id, + nonce, + timestamp_ns, + value, + }, + signer_wallet, + ) + .unwrap(); + ReceiptWithState::new(receipt) + } + + pub async fn store_receipt(pgpool: &PgPool, signed_receipt: &SignedReceipt) -> Result { + let encoded_signature = signed_receipt.signature.to_vec(); + + let record = sqlx::query!( + r#" + INSERT INTO scalar_tap_receipts (signer_address, signature, allocation_id, timestamp_ns, nonce, value) + VALUES ($1, $2, $3, $4, $5, $6) + RETURNING id + "#, + signed_receipt + .recover_signer(&TAP_EIP712_DOMAIN_SEPARATOR) + .unwrap() + .encode_hex::(), + encoded_signature, + signed_receipt.message.allocation_id.encode_hex::(), + BigDecimal::from(signed_receipt.message.timestamp_ns), + BigDecimal::from(signed_receipt.message.nonce), + BigDecimal::from(BigInt::from(signed_receipt.message.value)), + ) + .fetch_one(pgpool) + .await?; + + // id is BIGSERIAL, so it should be safe to cast to u64. + let id: u64 = record.id.try_into()?; + Ok(id) + } /// Insert a single receipt and retrieve it from the database using the adapter. /// The point here it to test the deserialization of large numbers. @@ -218,8 +273,7 @@ mod test { ); let received_receipt = - create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, u64::MAX, u64::MAX, u128::MAX) - .await; + create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, u64::MAX, u64::MAX, u128::MAX); // Storing the receipt store_receipt(&storage_adapter.pgpool, received_receipt.signed_receipt()) @@ -437,38 +491,29 @@ mod test { // Creating 10 receipts with timestamps 42 to 51 let mut received_receipt_vec = Vec::new(); for i in 0..10 { - received_receipt_vec.push( - create_received_receipt( - &ALLOCATION_ID_0, - &SIGNER.0, - i + 684, - i + 42, - (i + 124).into(), - ) - .await, - ); + received_receipt_vec.push(create_received_receipt( + &ALLOCATION_ID_0, + &SIGNER.0, + i + 684, + i + 42, + (i + 124).into(), + )); // Adding irrelevant receipts to make sure they are not retrieved - received_receipt_vec.push( - create_received_receipt( - &ALLOCATION_ID_IRRELEVANT, - &SIGNER.0, - i + 684, - i + 42, - (i + 124).into(), - ) - .await, - ); - received_receipt_vec.push( - create_received_receipt( - &ALLOCATION_ID_0, - &SENDER_IRRELEVANT.0, - i + 684, - i + 42, - (i + 124).into(), - ) - .await, - ); + received_receipt_vec.push(create_received_receipt( + &ALLOCATION_ID_IRRELEVANT, + &SIGNER.0, + i + 684, + i + 42, + (i + 124).into(), + )); + received_receipt_vec.push(create_received_receipt( + &ALLOCATION_ID_0, + &SENDER_IRRELEVANT.0, + i + 684, + i + 42, + (i + 124).into(), + )); } // Storing the receipts @@ -574,38 +619,29 @@ mod test { // Creating 10 receipts with timestamps 42 to 51 let mut received_receipt_vec = Vec::new(); for i in 0..10 { - received_receipt_vec.push( - create_received_receipt( - &ALLOCATION_ID_0, - &SIGNER.0, - i + 684, - i + 42, - (i + 124).into(), - ) - .await, - ); + received_receipt_vec.push(create_received_receipt( + &ALLOCATION_ID_0, + &SIGNER.0, + i + 684, + i + 42, + (i + 124).into(), + )); // Adding irrelevant receipts to make sure they are not retrieved - received_receipt_vec.push( - create_received_receipt( - &ALLOCATION_ID_IRRELEVANT, - &SIGNER.0, - i + 684, - i + 42, - (i + 124).into(), - ) - .await, - ); - received_receipt_vec.push( - create_received_receipt( - &ALLOCATION_ID_0, - &SENDER_IRRELEVANT.0, - i + 684, - i + 42, - (i + 124).into(), - ) - .await, - ); + received_receipt_vec.push(create_received_receipt( + &ALLOCATION_ID_IRRELEVANT, + &SIGNER.0, + i + 684, + i + 42, + (i + 124).into(), + )); + received_receipt_vec.push(create_received_receipt( + &ALLOCATION_ID_0, + &SENDER_IRRELEVANT.0, + i + 684, + i + 42, + (i + 124).into(), + )); } macro_rules! test_ranges{ diff --git a/tap-agent/src/tap/mod.rs b/tap-agent/src/tap/mod.rs index 3802cf8b7..25998ebb6 100644 --- a/tap-agent/src/tap/mod.rs +++ b/tap-agent/src/tap/mod.rs @@ -11,7 +11,7 @@ pub mod context; pub mod escrow_adapter; #[cfg(test)] -pub mod test_utils; +mod test_utils; pub async fn signers_trimmed( escrow_accounts: &Eventual, diff --git a/tap-agent/src/tap/test_utils.rs b/tap-agent/src/tap/test_utils.rs index a93e51df7..65af769fd 100644 --- a/tap-agent/src/tap/test_utils.rs +++ b/tap-agent/src/tap/test_utils.rs @@ -3,33 +3,16 @@ use std::str::FromStr; -use alloy_primitives::hex::ToHex; use alloy_sol_types::{eip712_domain, Eip712Domain}; -use anyhow::Result; -use bigdecimal::num_bigint::BigInt; use ethers_signers::{coins_bip39::English, LocalWallet, MnemonicBuilder, Signer}; use lazy_static::lazy_static; -use sqlx::{types::BigDecimal, PgPool}; -use tap_core::{ - rav::{ReceiptAggregateVoucher, SignedRAV}, - receipt::{Checking, Receipt, ReceiptWithState, SignedReceipt}, - signed_message::EIP712SignedMessage, -}; use thegraph::types::Address; lazy_static! { pub static ref ALLOCATION_ID_0: Address = Address::from_str("0xabababababababababababababababababababab").unwrap(); - pub static ref ALLOCATION_ID_1: Address = - Address::from_str("0xbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc").unwrap(); - pub static ref ALLOCATION_ID_2: Address = - Address::from_str("0xcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd").unwrap(); - pub static ref ALLOCATION_ID_IRRELEVANT: Address = - Address::from_str("0xbcdebcdebcdebcdebcdebcdebcdebcdebcdebcde").unwrap(); pub static ref SENDER: (LocalWallet, Address) = wallet(0); - pub static ref SENDER_IRRELEVANT: (LocalWallet, Address) = wallet(1); pub static ref SIGNER: (LocalWallet, Address) = wallet(2); - pub static ref INDEXER: (LocalWallet, Address) = wallet(3); pub static ref TAP_EIP712_DOMAIN_SEPARATOR: Eip712Domain = eip712_domain! { name: "TAP", version: "1", @@ -49,92 +32,3 @@ pub fn wallet(index: u32) -> (LocalWallet, Address) { let address = wallet.address(); (wallet, Address::from_slice(address.as_bytes())) } - -/// Fixture to generate a signed receipt using the wallet from `keys()` and the -/// given `query_id` and `value` -pub async fn create_received_receipt( - allocation_id: &Address, - signer_wallet: &LocalWallet, - nonce: u64, - timestamp_ns: u64, - value: u128, -) -> ReceiptWithState { - let receipt = EIP712SignedMessage::new( - &TAP_EIP712_DOMAIN_SEPARATOR, - Receipt { - allocation_id: *allocation_id, - nonce, - timestamp_ns, - value, - }, - signer_wallet, - ) - .unwrap(); - ReceiptWithState::new(receipt) -} - -/// Fixture to generate a RAV using the wallet from `keys()` -pub async fn create_rav( - allocation_id: Address, - signer_wallet: LocalWallet, - timestamp_ns: u64, - value_aggregate: u128, -) -> SignedRAV { - EIP712SignedMessage::new( - &TAP_EIP712_DOMAIN_SEPARATOR, - ReceiptAggregateVoucher { - allocationId: allocation_id, - timestampNs: timestamp_ns, - valueAggregate: value_aggregate, - }, - &signer_wallet, - ) - .unwrap() -} - -pub async fn store_receipt(pgpool: &PgPool, signed_receipt: &SignedReceipt) -> Result { - let encoded_signature = signed_receipt.signature.to_vec(); - - let record = sqlx::query!( - r#" - INSERT INTO scalar_tap_receipts (signer_address, signature, allocation_id, timestamp_ns, nonce, value) - VALUES ($1, $2, $3, $4, $5, $6) - RETURNING id - "#, - signed_receipt - .recover_signer(&TAP_EIP712_DOMAIN_SEPARATOR) - .unwrap() - .encode_hex::(), - encoded_signature, - signed_receipt.message.allocation_id.encode_hex::(), - BigDecimal::from(signed_receipt.message.timestamp_ns), - BigDecimal::from(signed_receipt.message.nonce), - BigDecimal::from(BigInt::from(signed_receipt.message.value)), - ) - .fetch_one(pgpool) - .await?; - - // id is BIGSERIAL, so it should be safe to cast to u64. - let id: u64 = record.id.try_into()?; - Ok(id) -} - -pub async fn store_rav(pgpool: &PgPool, signed_rav: SignedRAV, sender: Address) -> Result<()> { - let signature_bytes = signed_rav.signature.to_vec(); - - let _fut = sqlx::query!( - r#" - INSERT INTO scalar_tap_ravs (sender_address, signature, allocation_id, timestamp_ns, value_aggregate) - VALUES ($1, $2, $3, $4, $5) - "#, - sender.encode_hex::(), - signature_bytes, - signed_rav.message.allocationId.encode_hex::(), - BigDecimal::from(signed_rav.message.timestampNs), - BigDecimal::from(BigInt::from(signed_rav.message.valueAggregate)), - ) - .execute(pgpool) - .await?; - - Ok(()) -} diff --git a/tap-agent/tests/sender_account_tests.rs b/tap-agent/tests/sender_account_tests.rs new file mode 100644 index 000000000..d7a1aa05c --- /dev/null +++ b/tap-agent/tests/sender_account_tests.rs @@ -0,0 +1,551 @@ +// Copyright 2023-, GraphOps and Semiotic Labs. +// SPDX-License-Identifier: Apache-2.0 + +use alloy_primitives::hex::ToHex; +use alloy_primitives::Address; +use bigdecimal::{num_bigint::ToBigInt, ToPrimitive}; +use eventuals::Eventual; +use indexer_common::escrow_accounts::EscrowAccounts; +use indexer_common::subgraph_client::{DeploymentDetails, SubgraphClient}; +use indexer_tap_agent::agent::sender_account::{ + SenderAccount, SenderAccountArgs, SenderAccountMessage, +}; +use indexer_tap_agent::agent::sender_accounts_manager::NewReceiptNotification; +use indexer_tap_agent::agent::sender_allocation::SenderAllocationMessage; +use indexer_tap_agent::agent::unaggregated_receipts::UnaggregatedReceipts; +use indexer_tap_agent::config; +use ractor::{call, cast, Actor, ActorRef}; +use serde_json::json; +use sqlx::PgPool; +use std::collections::HashSet; +use std::str::FromStr; +use std::time::Duration; +use std::{collections::HashMap, sync::atomic::AtomicU32}; +use tap_aggregator::server::run_server; +use tap_core::{rav::ReceiptAggregateVoucher, signed_message::EIP712SignedMessage}; +use tokio::task::JoinHandle; +use wiremock::{ + matchers::{body_string_contains, method}, + Mock, MockServer, ResponseTemplate, +}; + +use test_utils::{ + create_received_receipt, store_receipt, ALLOCATION_ID_0, ALLOCATION_ID_1, ALLOCATION_ID_2, + INDEXER, SENDER, SIGNER, TAP_EIP712_DOMAIN_SEPARATOR, +}; + +mod test_utils; + +const DUMMY_URL: &str = "http://localhost:1234"; +const VALUE_PER_RECEIPT: u64 = 100; +const TRIGGER_VALUE: u64 = 500; +static PREFIX_ID: AtomicU32 = AtomicU32::new(0); + +// To help with testing from other modules. + +async fn create_sender_with_allocations( + pgpool: PgPool, + sender_aggregator_endpoint: String, + escrow_subgraph_endpoint: &str, +) -> (ActorRef, JoinHandle<()>, String) { + let config = Box::leak(Box::new(config::Cli { + config: None, + ethereum: config::Ethereum { + indexer_address: INDEXER.1, + }, + tap: config::Tap { + rav_request_trigger_value: TRIGGER_VALUE, + rav_request_timestamp_buffer_ms: 1, + rav_request_timeout_secs: 5, + ..Default::default() + }, + ..Default::default() + })); + + let escrow_subgraph = Box::leak(Box::new(SubgraphClient::new( + reqwest::Client::new(), + None, + DeploymentDetails::for_query_url(escrow_subgraph_endpoint).unwrap(), + ))); + + let escrow_accounts_eventual = Eventual::from_value(EscrowAccounts::new( + HashMap::from([(SENDER.1, 1000.into())]), + HashMap::from([(SENDER.1, vec![SIGNER.1])]), + )); + + let indexer_allocations = Eventual::from_value(HashSet::from([ + *ALLOCATION_ID_0, + *ALLOCATION_ID_1, + *ALLOCATION_ID_2, + ])); + + let prefix = format!( + "test-{}", + PREFIX_ID.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + ); + + let args = SenderAccountArgs { + config, + pgpool, + sender_id: SENDER.1, + escrow_accounts: escrow_accounts_eventual, + indexer_allocations, + escrow_subgraph, + domain_separator: TAP_EIP712_DOMAIN_SEPARATOR.clone(), + sender_aggregator_endpoint, + allocation_ids: HashSet::new(), + prefix: Some(prefix.clone()), + }; + + let (sender, handle) = SenderAccount::spawn(Some(prefix.clone()), SenderAccount, args) + .await + .unwrap(); + + // await for the allocations to be created + ractor::concurrency::sleep(Duration::from_millis(100)).await; + + (sender, handle, prefix) +} + +/// Test that the sender_account correctly ignores new receipt notifications with +/// an ID lower than the last receipt ID processed (be it from the DB or from a prior receipt +/// notification). +#[sqlx::test(migrations = "../migrations")] +async fn test_handle_new_receipt_notification(pgpool: PgPool) { + // Add receipts to the database. Before creating the sender and allocation so that it loads + // the receipts from the DB. + let mut expected_unaggregated_fees = 0u128; + for i in 10..20 { + let receipt = create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i, i.into()).await; + store_receipt(&pgpool, receipt.signed_receipt()) + .await + .unwrap(); + expected_unaggregated_fees += u128::from(i); + } + + let (sender, handle, prefix) = + create_sender_with_allocations(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL).await; + + let allocation = ActorRef::::where_is(format!( + "{prefix}:{sender}:{allocation_id}", + sender = SENDER.1, + allocation_id = *ALLOCATION_ID_0 + )) + .unwrap(); + + // Check that the sender's unaggregated fees are correct. + let allocation_tracker = call!(sender, SenderAccountMessage::GetAllocationTracker).unwrap(); + assert_eq!( + allocation_tracker.get_total_fee(), + expected_unaggregated_fees + ); + + let allocation_unaggregated_fees = + call!(allocation, SenderAllocationMessage::GetUnaggregatedReceipts).unwrap(); + + // Check that the allocation's unaggregated fees are correct. + assert_eq!( + allocation_unaggregated_fees.value, + expected_unaggregated_fees + ); + + // Send a new receipt notification that has a lower ID than the last loaded from the DB. + // The last ID in the DB should be 10, since we added 10 receipts to the empty receipts + // table + let new_receipt_notification = NewReceiptNotification { + allocation_id: *ALLOCATION_ID_0, + signer_address: SIGNER.1, + id: 10, + timestamp_ns: 19, + value: 19, + }; + allocation + .cast(SenderAllocationMessage::NewReceipt( + new_receipt_notification, + )) + .unwrap(); + ractor::concurrency::sleep(Duration::from_millis(10)).await; + + let allocation_unaggregated_fees = + call!(allocation, SenderAllocationMessage::GetUnaggregatedReceipts).unwrap(); + + // Check that the allocation's unaggregated fees have *not* increased. + assert_eq!( + allocation_unaggregated_fees.value, + expected_unaggregated_fees + ); + + // Check that the unaggregated fees have *not* increased. + let allocation_tracker = call!(sender, SenderAccountMessage::GetAllocationTracker).unwrap(); + assert_eq!( + allocation_tracker.get_total_fee(), + expected_unaggregated_fees + ); + + // Send a new receipt notification. + let new_receipt_notification = NewReceiptNotification { + allocation_id: *ALLOCATION_ID_0, + signer_address: SIGNER.1, + id: 30, + timestamp_ns: 20, + value: 20, + }; + allocation + .cast(SenderAllocationMessage::NewReceipt( + new_receipt_notification, + )) + .unwrap(); + ractor::concurrency::sleep(Duration::from_millis(10)).await; + + expected_unaggregated_fees += 20; + + let allocation_unaggregated_fees = + call!(allocation, SenderAllocationMessage::GetUnaggregatedReceipts).unwrap(); + + // Check that the allocation's unaggregated fees are correct. + assert_eq!( + allocation_unaggregated_fees.value, + expected_unaggregated_fees + ); + + // Check that the sender's unaggregated fees are correct. + let allocation_tracker = call!(sender, SenderAccountMessage::GetAllocationTracker).unwrap(); + assert_eq!( + allocation_tracker.get_total_fee(), + expected_unaggregated_fees + ); + + // Send a new receipt notification that has a lower ID than the previous one. + let new_receipt_notification = NewReceiptNotification { + allocation_id: *ALLOCATION_ID_0, + signer_address: SIGNER.1, + id: 25, + timestamp_ns: 19, + value: 19, + }; + allocation + .cast(SenderAllocationMessage::NewReceipt( + new_receipt_notification, + )) + .unwrap(); + + ractor::concurrency::sleep(Duration::from_millis(10)).await; + + let allocation_unaggregated_fees = + call!(allocation, SenderAllocationMessage::GetUnaggregatedReceipts).unwrap(); + + // Check that the allocation's unaggregated fees have *not* increased. + assert_eq!( + allocation_unaggregated_fees.value, + expected_unaggregated_fees + ); + + // Check that the unaggregated fees have *not* increased. + let allocation_tracker = call!(sender, SenderAccountMessage::GetAllocationTracker).unwrap(); + assert_eq!( + allocation_tracker.get_total_fee(), + expected_unaggregated_fees + ); + + sender.stop(None); + handle.await.unwrap(); +} + +#[sqlx::test(migrations = "../migrations")] +async fn test_rav_requester_auto(pgpool: PgPool) { + // Start a TAP aggregator server. + let (handle, aggregator_endpoint) = run_server( + 0, + SIGNER.0.clone(), + vec![SIGNER.1].into_iter().collect(), + TAP_EIP712_DOMAIN_SEPARATOR.clone(), + 100 * 1024, + 100 * 1024, + 1, + ) + .await + .unwrap(); + + // Start a mock graphql server using wiremock + let mock_server = MockServer::start().await; + + // Mock result for TAP redeem txs for (allocation, sender) pair. + mock_server + .register( + Mock::given(method("POST")) + .and(body_string_contains("transactions")) + .respond_with( + ResponseTemplate::new(200) + .set_body_json(json!({ "data": { "transactions": []}})), + ), + ) + .await; + + // Create a sender_account. + let (sender_account, sender_handle, prefix) = create_sender_with_allocations( + pgpool.clone(), + "http://".to_owned() + &aggregator_endpoint.to_string(), + &mock_server.uri(), + ) + .await; + + let allocation = ActorRef::::where_is(format!( + "{prefix}:{sender}:{allocation_id}", + sender = SENDER.1, + allocation_id = *ALLOCATION_ID_0 + )) + .unwrap(); + + // Add receipts to the database and call the `handle_new_receipt_notification` method + // correspondingly. + let mut total_value = 0; + let mut trigger_value = 0; + for i in 1..=10 { + // These values should be enough to trigger a RAV request at i == 7 since we set the + // `rav_request_trigger_value` to 100. + let value = (i + VALUE_PER_RECEIPT) as u128; + + let receipt = create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i + 1, value).await; + store_receipt(&pgpool, receipt.signed_receipt()) + .await + .unwrap(); + let new_receipt_notification = NewReceiptNotification { + allocation_id: *ALLOCATION_ID_0, + signer_address: SIGNER.1, + id: i, + timestamp_ns: i + 1, + value, + }; + allocation + .cast(SenderAllocationMessage::NewReceipt( + new_receipt_notification, + )) + .unwrap(); + + ractor::concurrency::sleep(Duration::from_millis(100)).await; + + total_value += value; + if total_value >= TRIGGER_VALUE as u128 && trigger_value == 0 { + trigger_value = total_value; + } + } + + ractor::concurrency::sleep(Duration::from_millis(10)).await; + + // Wait for the RAV requester to finish. + for _ in 0..100 { + let allocation_tracker = + call!(sender_account, SenderAccountMessage::GetAllocationTracker).unwrap(); + if allocation_tracker.get_total_fee() < trigger_value { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + + // Get the latest RAV from the database. + let latest_rav = sqlx::query!( + r#" + SELECT signature, allocation_id, timestamp_ns, value_aggregate + FROM scalar_tap_ravs + WHERE allocation_id = $1 AND sender_address = $2 + "#, + ALLOCATION_ID_0.encode_hex::(), + SENDER.1.encode_hex::() + ) + .fetch_optional(&pgpool) + .await + .unwrap() + .unwrap(); + + let latest_rav = EIP712SignedMessage { + message: ReceiptAggregateVoucher { + allocationId: Address::from_str(&latest_rav.allocation_id).unwrap(), + timestampNs: latest_rav.timestamp_ns.to_u64().unwrap(), + // Beware, BigDecimal::to_u128() actually uses to_u64() under the hood... + // So we're converting to BigInt to get a proper implementation of to_u128(). + valueAggregate: latest_rav + .value_aggregate + .to_bigint() + .map(|v| v.to_u128()) + .unwrap() + .unwrap(), + }, + signature: latest_rav.signature.as_slice().try_into().unwrap(), + }; + + // Check that the latest RAV value is correct. + assert!(latest_rav.message.valueAggregate >= trigger_value); + + // Check that the allocation's unaggregated fees value is reduced. + + let allocation_unaggregated_fees = + call!(allocation, SenderAllocationMessage::GetUnaggregatedReceipts).unwrap(); + + // Check that the allocation's unaggregated fees have *not* increased. + assert!(allocation_unaggregated_fees.value <= trigger_value); + + // Check that the sender's unaggregated fees value is reduced. + let allocation_tracker = + call!(sender_account, SenderAccountMessage::GetAllocationTracker).unwrap(); + assert!(allocation_tracker.get_total_fee() <= trigger_value); + + // Reset the total value and trigger value. + total_value = allocation_tracker.get_total_fee(); + trigger_value = 0; + + // Add more receipts + for i in 10..20 { + let value = (i + VALUE_PER_RECEIPT) as u128; + + let receipt = + create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i + 1, i.into()).await; + store_receipt(&pgpool, receipt.signed_receipt()) + .await + .unwrap(); + + let new_receipt_notification = NewReceiptNotification { + allocation_id: *ALLOCATION_ID_0, + signer_address: SIGNER.1, + id: i, + timestamp_ns: i + 1, + value, + }; + allocation + .cast(SenderAllocationMessage::NewReceipt( + new_receipt_notification, + )) + .unwrap(); + + ractor::concurrency::sleep(Duration::from_millis(10)).await; + + total_value += value; + if total_value >= TRIGGER_VALUE as u128 && trigger_value == 0 { + trigger_value = total_value; + } + } + + // Wait for the RAV requester to finish. + for _ in 0..100 { + let allocation_tracker = + call!(sender_account, SenderAccountMessage::GetAllocationTracker).unwrap(); + if allocation_tracker.get_total_fee() < trigger_value { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + + // Get the latest RAV from the database. + let latest_rav = sqlx::query!( + r#" + SELECT signature, allocation_id, timestamp_ns, value_aggregate + FROM scalar_tap_ravs + WHERE allocation_id = $1 AND sender_address = $2 + "#, + ALLOCATION_ID_0.encode_hex::(), + SENDER.1.encode_hex::() + ) + .fetch_optional(&pgpool) + .await + .unwrap() + .unwrap(); + + let latest_rav = EIP712SignedMessage { + message: ReceiptAggregateVoucher { + allocationId: Address::from_str(&latest_rav.allocation_id).unwrap(), + timestampNs: latest_rav.timestamp_ns.to_u64().unwrap(), + // Beware, BigDecimal::to_u128() actually uses to_u64() under the hood... + // So we're converting to BigInt to get a proper implementation of to_u128(). + valueAggregate: latest_rav + .value_aggregate + .to_bigint() + .map(|v| v.to_u128()) + .unwrap() + .unwrap(), + }, + signature: latest_rav.signature.as_slice().try_into().unwrap(), + }; + + // Check that the latest RAV value is correct. + + assert!(latest_rav.message.valueAggregate >= trigger_value); + + // Check that the allocation's unaggregated fees value is reduced. + + let allocation_unaggregated_fees = + call!(allocation, SenderAllocationMessage::GetUnaggregatedReceipts).unwrap(); + + // Check that the allocation's unaggregated fees have *not* increased. + assert!(allocation_unaggregated_fees.value <= trigger_value); + + // Check that the unaggregated fees value is reduced. + let allocation_tracker = + call!(sender_account, SenderAccountMessage::GetAllocationTracker).unwrap(); + assert!(allocation_tracker.get_total_fee() <= trigger_value); + + // Stop the TAP aggregator server. + handle.stop().unwrap(); + handle.stopped().await; + + sender_account.stop(None); + sender_handle.await.unwrap(); +} + +#[sqlx::test(migrations = "../migrations")] +async fn test_sender_unaggregated_fees(pgpool: PgPool) { + // Create a sender_account. + let (sender_account, handle, _prefix) = + create_sender_with_allocations(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL).await; + + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + + // Closure that adds a number of receipts to an allocation. + + let update_receipt_fees = + |sender_account: &ActorRef, allocation_id: Address, value: u128| { + cast!( + sender_account, + SenderAccountMessage::UpdateReceiptFees( + allocation_id, + UnaggregatedReceipts { + value, + ..Default::default() + }, + ) + ) + .unwrap(); + + value + }; + + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + + // Add receipts to the database for allocation_0 + let total_value_0 = update_receipt_fees(&sender_account, *ALLOCATION_ID_0, 90); + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + + // Add receipts to the database for allocation_1 + let total_value_1 = update_receipt_fees(&sender_account, *ALLOCATION_ID_1, 100); + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + + // Add receipts to the database for allocation_2 + let total_value_2 = update_receipt_fees(&sender_account, *ALLOCATION_ID_2, 80); + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + + let allocation_tracker = + call!(sender_account, SenderAccountMessage::GetAllocationTracker).unwrap(); + + // Get the heaviest allocation. + let heaviest_allocation = allocation_tracker.get_heaviest_allocation_id().unwrap(); + + // Check that the heaviest allocation is correct. + assert_eq!(heaviest_allocation, *ALLOCATION_ID_1); + + // Check that the sender's unaggregated fees value is correct. + assert_eq!( + allocation_tracker.get_total_fee(), + total_value_0 + total_value_1 + total_value_2 + ); + + sender_account.stop(None); + handle.await.unwrap(); +} diff --git a/tap-agent/tests/sender_allocation_tests.rs b/tap-agent/tests/sender_allocation_tests.rs index cf206ebfd..6a1b0bc5d 100644 --- a/tap-agent/tests/sender_allocation_tests.rs +++ b/tap-agent/tests/sender_allocation_tests.rs @@ -1,2 +1,238 @@ // Copyright 2023-, GraphOps and Semiotic Labs. // SPDX-License-Identifier: Apache-2.0 + +use std::collections::HashMap; + +use eventuals::Eventual; +use indexer_common::{ + escrow_accounts::EscrowAccounts, + subgraph_client::{DeploymentDetails, SubgraphClient}, +}; +use indexer_tap_agent::{ + agent::{ + sender_account::SenderAccountMessage, + sender_allocation::{SenderAllocation, SenderAllocationArgs, SenderAllocationMessage}, + }, + config, + tap::escrow_adapter::EscrowAdapter, +}; +use ractor::{call, Actor, ActorProcessingErr, ActorRef}; +use serde_json::json; +use sqlx::PgPool; +use tap_aggregator::server::run_server; + +use wiremock::{ + matchers::{body_string_contains, method}, + Mock, MockServer, ResponseTemplate, +}; + +mod test_utils; + +struct MockSenderAccount; + +#[async_trait::async_trait] +impl Actor for MockSenderAccount { + type Msg = SenderAccountMessage; + type State = (); + type Arguments = (); + + async fn pre_start( + &self, + _myself: ActorRef, + _allocation_ids: Self::Arguments, + ) -> std::result::Result { + Ok(()) + } +} + +use test_utils::{ + create_rav, create_received_receipt, store_rav, store_receipt, ALLOCATION_ID_0, INDEXER, + SENDER, SIGNER, TAP_EIP712_DOMAIN_SEPARATOR, +}; + +const DUMMY_URL: &str = "http://localhost:1234"; + +async fn create_sender_allocation( + pgpool: PgPool, + sender_aggregator_endpoint: String, + escrow_subgraph_endpoint: &str, +) -> ActorRef { + let config = Box::leak(Box::new(config::Cli { + config: None, + ethereum: config::Ethereum { + indexer_address: INDEXER.1, + }, + tap: config::Tap { + rav_request_trigger_value: 100, + rav_request_timestamp_buffer_ms: 1, + rav_request_timeout_secs: 5, + ..Default::default() + }, + ..Default::default() + })); + + let escrow_subgraph = Box::leak(Box::new(SubgraphClient::new( + reqwest::Client::new(), + None, + DeploymentDetails::for_query_url(escrow_subgraph_endpoint).unwrap(), + ))); + + let escrow_accounts_eventual = Eventual::from_value(EscrowAccounts::new( + HashMap::from([(SENDER.1, 1000.into())]), + HashMap::from([(SENDER.1, vec![SIGNER.1])]), + )); + + let escrow_adapter = EscrowAdapter::new(escrow_accounts_eventual.clone(), SENDER.1); + + let (sender_account_ref, _join_handle) = MockSenderAccount::spawn(None, MockSenderAccount, ()) + .await + .unwrap(); + + let args = SenderAllocationArgs { + config, + pgpool: pgpool.clone(), + allocation_id: *ALLOCATION_ID_0, + sender: SENDER.1, + escrow_accounts: escrow_accounts_eventual, + escrow_subgraph, + escrow_adapter, + domain_separator: TAP_EIP712_DOMAIN_SEPARATOR.clone(), + sender_aggregator_endpoint, + sender_account_ref, + }; + + let (allocation_ref, _join_handle) = SenderAllocation::spawn(None, SenderAllocation, args) + .await + .unwrap(); + + allocation_ref +} + +/// Test that the sender_allocation correctly updates the unaggregated fees from the +/// database when there is no RAV in the database. +/// +/// The sender_allocation should consider all receipts found for the allocation and +/// sender. +#[sqlx::test(migrations = "../migrations")] +async fn test_update_unaggregated_fees_no_rav(pgpool: PgPool) { + // Add receipts to the database. + for i in 1..10 { + let receipt = create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i, i.into()).await; + store_receipt(&pgpool, receipt.signed_receipt()) + .await + .unwrap(); + } + + let sender_allocation = + create_sender_allocation(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL).await; + + // Get total_unaggregated_fees + let total_unaggregated_fees = call!( + sender_allocation, + SenderAllocationMessage::GetUnaggregatedReceipts + ) + .unwrap(); + + // Check that the unaggregated fees are correct. + assert_eq!(total_unaggregated_fees.value, 45u128); +} + +/// Test that the sender_allocation correctly updates the unaggregated fees from the +/// database when there is a RAV in the database as well as receipts which timestamp are lesser +/// and greater than the RAV's timestamp. +/// +/// The sender_allocation should only consider receipts with a timestamp greater +/// than the RAV's timestamp. +#[sqlx::test(migrations = "../migrations")] +async fn test_update_unaggregated_fees_with_rav(pgpool: PgPool) { + // Add the RAV to the database. + // This RAV has timestamp 4. The sender_allocation should only consider receipts + // with a timestamp greater than 4. + let signed_rav = create_rav(*ALLOCATION_ID_0, SIGNER.0.clone(), 4, 10).await; + store_rav(&pgpool, signed_rav, SENDER.1).await.unwrap(); + + // Add receipts to the database. + for i in 1..10 { + let receipt = create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i, i.into()).await; + store_receipt(&pgpool, receipt.signed_receipt()) + .await + .unwrap(); + } + + let sender_allocation = + create_sender_allocation(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL).await; + + // Get total_unaggregated_fees + let total_unaggregated_fees = call!( + sender_allocation, + SenderAllocationMessage::GetUnaggregatedReceipts + ) + .unwrap(); + + // Check that the unaggregated fees are correct. + assert_eq!(total_unaggregated_fees.value, 35u128); +} + +#[sqlx::test(migrations = "../migrations")] +async fn test_rav_requester_manual(pgpool: PgPool) { + // Start a TAP aggregator server. + let (handle, aggregator_endpoint) = run_server( + 0, + SIGNER.0.clone(), + vec![SIGNER.1].into_iter().collect(), + TAP_EIP712_DOMAIN_SEPARATOR.clone(), + 100 * 1024, + 100 * 1024, + 1, + ) + .await + .unwrap(); + + // Start a mock graphql server using wiremock + let mock_server = MockServer::start().await; + + // Mock result for TAP redeem txs for (allocation, sender) pair. + mock_server + .register( + Mock::given(method("POST")) + .and(body_string_contains("transactions")) + .respond_with( + ResponseTemplate::new(200) + .set_body_json(json!({ "data": { "transactions": []}})), + ), + ) + .await; + + // Add receipts to the database. + for i in 0..10 { + let receipt = + create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i + 1, i.into()).await; + store_receipt(&pgpool, receipt.signed_receipt()) + .await + .unwrap(); + } + + // Create a sender_allocation. + let sender_allocation = create_sender_allocation( + pgpool.clone(), + "http://".to_owned() + &aggregator_endpoint.to_string(), + &mock_server.uri(), + ) + .await; + + // Trigger a RAV request manually. + + // Get total_unaggregated_fees + let total_unaggregated_fees = call!( + sender_allocation, + SenderAllocationMessage::TriggerRAVRequest + ) + .unwrap(); + + // Check that the unaggregated fees are correct. + assert_eq!(total_unaggregated_fees.value, 0u128); + + // Stop the TAP aggregator server. + handle.stop().unwrap(); + handle.stopped().await; +} diff --git a/tap-agent/tests/sender_manager_tests.rs b/tap-agent/tests/sender_manager_tests.rs new file mode 100644 index 000000000..46fbefc23 --- /dev/null +++ b/tap-agent/tests/sender_manager_tests.rs @@ -0,0 +1,152 @@ +// Copyright 2023-, GraphOps and Semiotic Labs. +// SPDX-License-Identifier: Apache-2.0 + +use std::{collections::HashMap, str::FromStr, vec}; + +use alloy_primitives::Address; +use ethereum_types::U256; +use eventuals::Eventual; +use indexer_common::{ + allocations::Allocation, + escrow_accounts::EscrowAccounts, + prelude::{AllocationStatus, SubgraphDeployment}, + subgraph_client::{DeploymentDetails, SubgraphClient}, +}; +use indexer_tap_agent::{ + agent::{ + sender_account::SenderAccountMessage, + sender_accounts_manager::{SenderAccountsManager, SenderAccountsManagerArgs}, + sender_allocation::SenderAllocationMessage, + }, + config, +}; +use ractor::{Actor, ActorRef, ActorStatus}; +use serde_json::json; +use sqlx::PgPool; +use thegraph::types::DeploymentId; +use wiremock::{ + matchers::{body_string_contains, method}, + Mock, MockServer, ResponseTemplate, +}; + +mod test_utils; +use test_utils::{INDEXER, SENDER, SIGNER, TAP_EIP712_DOMAIN_SEPARATOR}; + +#[sqlx::test(migrations = "../migrations")] +async fn test_sender_account_creation_and_eol(pgpool: PgPool) { + let config = Box::leak(Box::new(config::Cli { + config: None, + ethereum: config::Ethereum { + indexer_address: INDEXER.1, + }, + tap: config::Tap { + rav_request_trigger_value: 100, + rav_request_timestamp_buffer_ms: 1, + ..Default::default() + }, + ..Default::default() + })); + + let (mut indexer_allocations_writer, indexer_allocations_eventual) = + Eventual::>::new(); + indexer_allocations_writer.write(HashMap::new()); + + let (mut escrow_accounts_writer, escrow_accounts_eventual) = Eventual::::new(); + escrow_accounts_writer.write(EscrowAccounts::default()); + + // Mock escrow subgraph. + let mock_server = MockServer::start().await; + mock_server + .register( + Mock::given(method("POST")) + .and(body_string_contains("transactions")) + .respond_with( + ResponseTemplate::new(200) + .set_body_json(json!({ "data": { "transactions": []}})), + ), + ) + .await; + let escrow_subgraph = Box::leak(Box::new(SubgraphClient::new( + reqwest::Client::new(), + None, + DeploymentDetails::for_query_url(&mock_server.uri()).unwrap(), + ))); + + let args = SenderAccountsManagerArgs { + config, + domain_separator: TAP_EIP712_DOMAIN_SEPARATOR.clone(), + pgpool: pgpool.clone(), + indexer_allocations: indexer_allocations_eventual, + escrow_accounts: escrow_accounts_eventual, + escrow_subgraph, + sender_aggregator_endpoints: HashMap::from([( + SENDER.1, + String::from("http://localhost:8000"), + )]), + }; + SenderAccountsManager::spawn(None, SenderAccountsManager, args) + .await + .unwrap(); + + let allocation_id = Address::from_str("0xdd975e30aafebb143e54d215db8a3e8fd916a701").unwrap(); + + // Add an allocation to the indexer_allocations Eventual. + indexer_allocations_writer.write(HashMap::from([( + allocation_id, + Allocation { + id: allocation_id, + indexer: INDEXER.1, + allocated_tokens: U256::from_str("601726452999999979510903").unwrap(), + created_at_block_hash: + "0x99d3fbdc0105f7ccc0cd5bb287b82657fe92db4ea8fb58242dafb90b1c6e2adf".to_string(), + created_at_epoch: 953, + closed_at_epoch: None, + subgraph_deployment: SubgraphDeployment { + id: DeploymentId::from_str( + "0xcda7fa0405d6fd10721ed13d18823d24b535060d8ff661f862b26c23334f13bf", + ) + .unwrap(), + denied_at: Some(0), + }, + status: AllocationStatus::Null, + closed_at_epoch_start_block_hash: None, + previous_epoch_start_block_hash: None, + poi: None, + query_fee_rebates: None, + query_fees_collected: None, + }, + )])); + + // Add an escrow account to the escrow_accounts Eventual. + escrow_accounts_writer.write(EscrowAccounts::new( + HashMap::from([(SENDER.1, 1000.into())]), + HashMap::from([(SENDER.1, vec![SIGNER.1])]), + )); + + // Wait for the SenderAccount to be created. + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + + // Check that the SenderAccount was created. + let sender_account = ActorRef::::where_is(SENDER.1.to_string()).unwrap(); + assert_eq!(sender_account.get_status(), ActorStatus::Running); + + let sender_allocation_id = format!("{}:{}", SENDER.1, allocation_id); + let sender_allocation = + ActorRef::::where_is(sender_allocation_id).unwrap(); + assert_eq!(sender_allocation.get_status(), ActorStatus::Running); + + // Remove the escrow account from the escrow_accounts Eventual. + escrow_accounts_writer.write(EscrowAccounts::default()); + + // Wait a bit + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + + // Check that the Sender's allocation moved from active to ineligible. + + assert_eq!(sender_account.get_status(), ActorStatus::Stopped); + assert_eq!(sender_allocation.get_status(), ActorStatus::Stopped); + + let sender_account = ActorRef::::where_is(SENDER.1.to_string()); + + assert!(sender_account.is_none()); +} diff --git a/tap-agent/tests/test_utils.rs b/tap-agent/tests/test_utils.rs new file mode 100644 index 000000000..9ddd7d11f --- /dev/null +++ b/tap-agent/tests/test_utils.rs @@ -0,0 +1,140 @@ +// Copyright 2023-, GraphOps and Semiotic Labs. +// SPDX-License-Identifier: Apache-2.0 + +use std::str::FromStr; + +use alloy_primitives::hex::ToHex; +use alloy_sol_types::{eip712_domain, Eip712Domain}; +use anyhow::Result; +use bigdecimal::num_bigint::BigInt; +use ethers_signers::{coins_bip39::English, LocalWallet, MnemonicBuilder, Signer}; +use lazy_static::lazy_static; +use sqlx::{types::BigDecimal, PgPool}; +use tap_core::{ + rav::{ReceiptAggregateVoucher, SignedRAV}, + receipt::{Checking, Receipt, ReceiptWithState, SignedReceipt}, + signed_message::EIP712SignedMessage, +}; +use thegraph::types::Address; + +lazy_static! { + pub static ref ALLOCATION_ID_0: Address = + Address::from_str("0xabababababababababababababababababababab").unwrap(); + pub static ref ALLOCATION_ID_1: Address = + Address::from_str("0xbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc").unwrap(); + pub static ref ALLOCATION_ID_2: Address = + Address::from_str("0xcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd").unwrap(); + pub static ref ALLOCATION_ID_IRRELEVANT: Address = + Address::from_str("0xbcdebcdebcdebcdebcdebcdebcdebcdebcdebcde").unwrap(); + pub static ref SENDER: (LocalWallet, Address) = wallet(0); + pub static ref SENDER_IRRELEVANT: (LocalWallet, Address) = wallet(1); + pub static ref SIGNER: (LocalWallet, Address) = wallet(2); + pub static ref INDEXER: (LocalWallet, Address) = wallet(3); + pub static ref TAP_EIP712_DOMAIN_SEPARATOR: Eip712Domain = eip712_domain! { + name: "TAP", + version: "1", + chain_id: 1, + verifying_contract: Address:: from([0x11u8; 20]), + }; +} + +/// Fixture to generate a wallet and address +fn wallet(index: u32) -> (LocalWallet, Address) { + let wallet: LocalWallet = MnemonicBuilder::::default() + .phrase("abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon about") + .index(index) + .unwrap() + .build() + .unwrap(); + let address = wallet.address(); + (wallet, Address::from_slice(address.as_bytes())) +} + +/// Fixture to generate a signed receipt using the wallet from `keys()` and the +/// given `query_id` and `value` +pub async fn create_received_receipt( + allocation_id: &Address, + signer_wallet: &LocalWallet, + nonce: u64, + timestamp_ns: u64, + value: u128, +) -> ReceiptWithState { + let receipt = EIP712SignedMessage::new( + &TAP_EIP712_DOMAIN_SEPARATOR, + Receipt { + allocation_id: *allocation_id, + nonce, + timestamp_ns, + value, + }, + signer_wallet, + ) + .unwrap(); + ReceiptWithState::new(receipt) +} + +/// Fixture to generate a RAV using the wallet from `keys()` +pub async fn create_rav( + allocation_id: Address, + signer_wallet: LocalWallet, + timestamp_ns: u64, + value_aggregate: u128, +) -> SignedRAV { + EIP712SignedMessage::new( + &TAP_EIP712_DOMAIN_SEPARATOR, + ReceiptAggregateVoucher { + allocationId: allocation_id, + timestampNs: timestamp_ns, + valueAggregate: value_aggregate, + }, + &signer_wallet, + ) + .unwrap() +} + +pub async fn store_receipt(pgpool: &PgPool, signed_receipt: &SignedReceipt) -> Result { + let encoded_signature = signed_receipt.signature.to_vec(); + + let record = sqlx::query!( + r#" + INSERT INTO scalar_tap_receipts (signer_address, signature, allocation_id, timestamp_ns, nonce, value) + VALUES ($1, $2, $3, $4, $5, $6) + RETURNING id + "#, + signed_receipt + .recover_signer(&TAP_EIP712_DOMAIN_SEPARATOR) + .unwrap() + .encode_hex::(), + encoded_signature, + signed_receipt.message.allocation_id.encode_hex::(), + BigDecimal::from(signed_receipt.message.timestamp_ns), + BigDecimal::from(signed_receipt.message.nonce), + BigDecimal::from(BigInt::from(signed_receipt.message.value)), + ) + .fetch_one(pgpool) + .await?; + + // id is BIGSERIAL, so it should be safe to cast to u64. + let id: u64 = record.id.try_into()?; + Ok(id) +} + +pub async fn store_rav(pgpool: &PgPool, signed_rav: SignedRAV, sender: Address) -> Result<()> { + let signature_bytes = signed_rav.signature.to_vec(); + + let _fut = sqlx::query!( + r#" + INSERT INTO scalar_tap_ravs (sender_address, signature, allocation_id, timestamp_ns, value_aggregate) + VALUES ($1, $2, $3, $4, $5) + "#, + sender.encode_hex::(), + signature_bytes, + signed_rav.message.allocationId.encode_hex::(), + BigDecimal::from(signed_rav.message.timestampNs), + BigDecimal::from(BigInt::from(signed_rav.message.valueAggregate)), + ) + .execute(pgpool) + .await?; + + Ok(()) +} From b6c9242bdd4fc1927ab7081287302b9820e09022 Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Tue, 2 Apr 2024 18:05:38 -0300 Subject: [PATCH 18/41] test: add more tests to sender_allocation Signed-off-by: Gustavo Inacio --- tap-agent/src/agent/sender_account.rs | 18 + tap-agent/src/agent/sender_allocation.rs | 499 ++++++++++++++++--- tap-agent/src/agent/unaggregated_receipts.rs | 2 +- tap-agent/src/tap/context.rs | 2 - tap-agent/src/tap/context/rav.rs | 23 +- tap-agent/src/tap/context/receipt.rs | 57 +-- tap-agent/src/tap/mod.rs | 2 +- tap-agent/src/tap/test_utils.rs | 105 ++++ tap-agent/tests/sender_allocation_tests.rs | 238 --------- 9 files changed, 558 insertions(+), 388 deletions(-) delete mode 100644 tap-agent/tests/sender_allocation_tests.rs diff --git a/tap-agent/src/agent/sender_account.rs b/tap-agent/src/agent/sender_account.rs index 7f13d0156..202b1250b 100644 --- a/tap-agent/src/agent/sender_account.rs +++ b/tap-agent/src/agent/sender_account.rs @@ -21,6 +21,7 @@ use crate::{ tap::escrow_adapter::EscrowAdapter, }; +#[derive(Debug)] pub enum SenderAccountMessage { CreateSenderAllocation(Address), UpdateAllocationIds(HashSet
), @@ -229,6 +230,23 @@ impl Actor for SenderAccount { #[cfg(test)] mod tests { + use super::SenderAccountMessage; + + // we implement the PartialEq and Eq traits for SenderAccountMessage to be able to compare + impl Eq for SenderAccountMessage {} + + impl PartialEq for SenderAccountMessage { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Self::CreateSenderAllocation(l0), Self::CreateSenderAllocation(r0)) => l0 == r0, + (Self::UpdateAllocationIds(l0), Self::UpdateAllocationIds(r0)) => l0 == r0, + (Self::UpdateReceiptFees(l0, l1), Self::UpdateReceiptFees(r0, r1)) => { + l0 == r0 && l1 == r1 + } + _ => core::mem::discriminant(self) == core::mem::discriminant(other), + } + } + } #[test] fn test_update_allocation_ids() {} diff --git a/tap-agent/src/agent/sender_allocation.rs b/tap-agent/src/agent/sender_allocation.rs index 8f6c3c403..c16081b13 100644 --- a/tap-agent/src/agent/sender_allocation.rs +++ b/tap-agent/src/agent/sender_allocation.rs @@ -81,56 +81,11 @@ impl Actor for SenderAllocation { async fn pre_start( &self, _myself: ActorRef, - SenderAllocationArgs { - config, - pgpool, - allocation_id, - sender, - escrow_accounts, - escrow_subgraph, - escrow_adapter, - domain_separator, - sender_aggregator_endpoint, - sender_account_ref, - }: Self::Arguments, + args: Self::Arguments, ) -> std::result::Result { - let required_checks: Vec> = vec![ - Arc::new(AllocationId::new( - sender, - allocation_id, - escrow_subgraph, - config, - )), - Arc::new(Signature::new( - domain_separator.clone(), - escrow_accounts.clone(), - )), - ]; - let context = TapAgentContext::new( - pgpool.clone(), - allocation_id, - sender, - escrow_accounts.clone(), - escrow_adapter, - ); - let tap_manager = TapManager::new( - domain_separator.clone(), - context, - Checks::new(required_checks), - ); - - let mut state = SenderAllocationState { - pgpool, - tap_manager, - allocation_id, - sender, - sender_aggregator_endpoint, - config, - escrow_accounts, - domain_separator, - sender_account_ref: sender_account_ref.clone(), - unaggregated_fees: UnaggregatedReceipts::default(), - }; + let sender_account_ref = args.sender_account_ref.clone(); + let allocation_id = args.allocation_id; + let mut state = SenderAllocationState::new(args); state.unaggregated_fees = state.calculate_unaggregated_fee().await?; sender_account_ref.cast(SenderAccountMessage::UpdateReceiptFees( @@ -229,6 +184,59 @@ impl Actor for SenderAllocation { } impl SenderAllocationState { + fn new( + SenderAllocationArgs { + config, + pgpool, + allocation_id, + sender, + escrow_accounts, + escrow_subgraph, + escrow_adapter, + domain_separator, + sender_aggregator_endpoint, + sender_account_ref, + }: SenderAllocationArgs, + ) -> Self { + let required_checks: Vec> = vec![ + Arc::new(AllocationId::new( + sender, + allocation_id, + escrow_subgraph, + config, + )), + Arc::new(Signature::new( + domain_separator.clone(), + escrow_accounts.clone(), + )), + ]; + let context = TapAgentContext::new( + pgpool.clone(), + allocation_id, + sender, + escrow_accounts.clone(), + escrow_adapter, + ); + let tap_manager = TapManager::new( + domain_separator.clone(), + context, + Checks::new(required_checks), + ); + + Self { + pgpool, + tap_manager, + allocation_id, + sender, + sender_aggregator_endpoint, + config, + escrow_accounts, + domain_separator, + sender_account_ref: sender_account_ref.clone(), + unaggregated_fees: UnaggregatedReceipts::default(), + } + } + /// Delete obsolete receipts in the DB w.r.t. the last RAV in DB, then update the tap manager /// with the latest unaggregated fees from the database. async fn calculate_unaggregated_fee(&self) -> Result { @@ -475,26 +483,307 @@ impl SenderAllocationState { #[cfg(test)] mod tests { + use std::{ + collections::HashMap, + sync::{Arc, Mutex}, + }; + + use eventuals::Eventual; + use indexer_common::{ + escrow_accounts::EscrowAccounts, + subgraph_client::{DeploymentDetails, SubgraphClient}, + }; + use ractor::{call, cast, concurrency::JoinHandle, Actor, ActorProcessingErr, ActorRef}; + use serde_json::json; + use sqlx::PgPool; + use tap_aggregator::server::run_server; + use wiremock::{ + matchers::{body_string_contains, method}, + Mock, MockServer, ResponseTemplate, + }; + + use crate::{ + agent::{ + sender_account::SenderAccountMessage, sender_accounts_manager::NewReceiptNotification, + unaggregated_receipts::UnaggregatedReceipts, + }, + config, + tap::{ + escrow_adapter::EscrowAdapter, + test_utils::{ + create_rav, create_received_receipt, store_rav, store_receipt, ALLOCATION_ID_0, + INDEXER, SENDER, SIGNER, TAP_EIP712_DOMAIN_SEPARATOR, + }, + }, + }; + + use super::{ + SenderAllocation, SenderAllocationArgs, SenderAllocationMessage, SenderAllocationState, + }; + + const DUMMY_URL: &str = "http://localhost:1234"; + + struct MockSenderAccount { + last_message_emitted: Arc>>, + } - #[test] - fn test_create_sender_allocation() { - // create an allocation + #[async_trait::async_trait] + impl Actor for MockSenderAccount { + type Msg = SenderAccountMessage; + type State = (); + type Arguments = (); + + async fn pre_start( + &self, + _myself: ActorRef, + _allocation_ids: Self::Arguments, + ) -> std::result::Result { + Ok(()) + } - // expect nothing fails? + async fn handle( + &self, + _myself: ActorRef, + message: Self::Msg, + _state: &mut Self::State, + ) -> std::result::Result<(), ActorProcessingErr> { + *self.last_message_emitted.lock().unwrap() = Some(message); + Ok(()) + } } - #[test] - fn test_receive_new_receipt() { + async fn create_mock_sender_account() -> ( + Arc>>, + ActorRef, + JoinHandle<()>, + ) { + let last_message_emitted = Arc::new(Mutex::new(None)); + + let (sender_account, join_handle) = MockSenderAccount::spawn( + None, + MockSenderAccount { + last_message_emitted: last_message_emitted.clone(), + }, + (), + ) + .await + .unwrap(); + (last_message_emitted, sender_account, join_handle) + } + + async fn create_sender_allocation_args( + pgpool: PgPool, + sender_aggregator_endpoint: String, + escrow_subgraph_endpoint: &str, + sender_account: Option>, + ) -> SenderAllocationArgs { + let config = Box::leak(Box::new(config::Cli { + config: None, + ethereum: config::Ethereum { + indexer_address: INDEXER.1, + }, + tap: config::Tap { + rav_request_trigger_value: 100, + rav_request_timestamp_buffer_ms: 1, + rav_request_timeout_secs: 5, + ..Default::default() + }, + ..Default::default() + })); + + let escrow_subgraph = Box::leak(Box::new(SubgraphClient::new( + reqwest::Client::new(), + None, + DeploymentDetails::for_query_url(escrow_subgraph_endpoint).unwrap(), + ))); + + let escrow_accounts_eventual = Eventual::from_value(EscrowAccounts::new( + HashMap::from([(SENDER.1, 1000.into())]), + HashMap::from([(SENDER.1, vec![SIGNER.1])]), + )); + + let escrow_adapter = EscrowAdapter::new(escrow_accounts_eventual.clone(), SENDER.1); + + let sender_account_ref = match sender_account { + Some(sender) => sender, + None => create_mock_sender_account().await.1, + }; + + SenderAllocationArgs { + config, + pgpool: pgpool.clone(), + allocation_id: *ALLOCATION_ID_0, + sender: SENDER.1, + escrow_accounts: escrow_accounts_eventual, + escrow_subgraph, + escrow_adapter, + domain_separator: TAP_EIP712_DOMAIN_SEPARATOR.clone(), + sender_aggregator_endpoint, + sender_account_ref, + } + } + + async fn create_sender_allocation( + pgpool: PgPool, + sender_aggregator_endpoint: String, + escrow_subgraph_endpoint: &str, + sender_account: Option>, + ) -> ActorRef { + let args = create_sender_allocation_args( + pgpool, + sender_aggregator_endpoint, + escrow_subgraph_endpoint, + sender_account, + ) + .await; + + let (allocation_ref, _join_handle) = SenderAllocation::spawn(None, SenderAllocation, args) + .await + .unwrap(); + + allocation_ref + } + + #[sqlx::test(migrations = "../migrations")] + async fn should_update_unaggregated_fees_on_start(pgpool: PgPool) { + // Add receipts to the database. + for i in 1..10 { + let receipt = create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i, i.into()); + store_receipt(&pgpool, receipt.signed_receipt()) + .await + .unwrap(); + } + + let sender_allocation = + create_sender_allocation(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL, None).await; + + // Get total_unaggregated_fees + let total_unaggregated_fees = call!( + sender_allocation, + SenderAllocationMessage::GetUnaggregatedReceipts + ) + .unwrap(); + + // Check that the unaggregated fees are correct. + assert_eq!(total_unaggregated_fees.value, 45u128); + } + + #[sqlx::test(migrations = "../migrations")] + fn test_receive_new_receipt(pgpool: PgPool) { + let (last_message_emitted, sender_account, _join_handle) = + create_mock_sender_account().await; + + let sender_allocation = create_sender_allocation( + pgpool.clone(), + DUMMY_URL.to_string(), + DUMMY_URL, + Some(sender_account), + ) + .await; + // should validate with id less than last_id + cast!( + sender_allocation, + SenderAllocationMessage::NewReceipt(NewReceiptNotification { + id: 0, + value: 10, + allocation_id: *ALLOCATION_ID_0, + signer_address: SIGNER.1, + timestamp_ns: 0, + }) + ) + .unwrap(); + + cast!( + sender_allocation, + SenderAllocationMessage::NewReceipt(NewReceiptNotification { + id: 1, + value: 20, + allocation_id: *ALLOCATION_ID_0, + signer_address: SIGNER.1, + timestamp_ns: 0, + }) + ) + .unwrap(); + + tokio::time::sleep(std::time::Duration::from_millis(10)).await; // should emit update aggregate fees message to sender account + let expected_message = SenderAccountMessage::UpdateReceiptFees( + *ALLOCATION_ID_0, + UnaggregatedReceipts { + last_id: 1, + value: 20, + }, + ); + assert_eq!( + *last_message_emitted.lock().unwrap(), + Some(expected_message) + ); } - #[test] - fn test_trigger_rav_request() { - // should rav request + #[sqlx::test(migrations = "../migrations")] + fn test_trigger_rav_request(pgpool: PgPool) { + // Start a TAP aggregator server. + let (handle, aggregator_endpoint) = run_server( + 0, + SIGNER.0.clone(), + vec![SIGNER.1].into_iter().collect(), + TAP_EIP712_DOMAIN_SEPARATOR.clone(), + 100 * 1024, + 100 * 1024, + 1, + ) + .await + .unwrap(); + + // Start a mock graphql server using wiremock + let mock_server = MockServer::start().await; + + // Mock result for TAP redeem txs for (allocation, sender) pair. + mock_server + .register( + Mock::given(method("POST")) + .and(body_string_contains("transactions")) + .respond_with( + ResponseTemplate::new(200) + .set_body_json(json!({ "data": { "transactions": []}})), + ), + ) + .await; + + // Add receipts to the database. + for i in 0..10 { + let receipt = create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i + 1, i.into()); + store_receipt(&pgpool, receipt.signed_receipt()) + .await + .unwrap(); + } - // should return correct unaggregated fees + // Create a sender_allocation. + let sender_allocation = create_sender_allocation( + pgpool.clone(), + "http://".to_owned() + &aggregator_endpoint.to_string(), + &mock_server.uri(), + None, + ) + .await; + + // Trigger a RAV request manually. + + // Get total_unaggregated_fees + let total_unaggregated_fees = call!( + sender_allocation, + SenderAllocationMessage::TriggerRAVRequest + ) + .unwrap(); + + // Check that the unaggregated fees are correct. + assert_eq!(total_unaggregated_fees.value, 0u128); + + // Stop the TAP aggregator server. + handle.stop().unwrap(); + handle.stopped().await; } #[test] @@ -512,29 +801,97 @@ mod tests { // check if message is sent to sender account } - #[test] - fn test_store_failed_rav() { - // check database if failed rav is stored + #[sqlx::test(migrations = "../migrations")] + async fn should_return_unaggregated_fees_without_rav(pgpool: PgPool) { + let args = + create_sender_allocation_args(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL, None) + .await; + let state = SenderAllocationState::new(args); + + // Add receipts to the database. + for i in 1..10 { + let receipt = create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i, i.into()); + store_receipt(&pgpool, receipt.signed_receipt()) + .await + .unwrap(); + } + + // calculate unaggregated fee + let total_unaggregated_fees = state.calculate_unaggregated_fee().await.unwrap(); + + // Check that the unaggregated fees are correct. + assert_eq!(total_unaggregated_fees.value, 45u128); } - #[test] - fn test_calculate_unaggregated_fee() { - // add a bunch of receipts + /// Test that the sender_allocation correctly updates the unaggregated fees from the + /// database when there is a RAV in the database as well as receipts which timestamp are lesser + /// and greater than the RAV's timestamp. + /// + /// The sender_allocation should only consider receipts with a timestamp greater + /// than the RAV's timestamp. + #[sqlx::test(migrations = "../migrations")] + async fn should_return_unaggregated_fees_with_rav(pgpool: PgPool) { + let args = + create_sender_allocation_args(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL, None) + .await; + let state = SenderAllocationState::new(args); + + // Add the RAV to the database. + // This RAV has timestamp 4. The sender_allocation should only consider receipts + // with a timestamp greater than 4. + let signed_rav = create_rav(*ALLOCATION_ID_0, SIGNER.0.clone(), 4, 10); + store_rav(&pgpool, signed_rav, SENDER.1).await.unwrap(); + + // Add receipts to the database. + for i in 1..10 { + let receipt = create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i, i.into()); + store_receipt(&pgpool, receipt.signed_receipt()) + .await + .unwrap(); + } - // calculate unaggregated fee + let total_unaggregated_fees = state.calculate_unaggregated_fee().await.unwrap(); - // check if it's correct + // Check that the unaggregated fees are correct. + assert_eq!(total_unaggregated_fees.value, 35u128); } - #[test] - fn test_store_invalid_receipts() { + #[sqlx::test(migrations = "../migrations")] + async fn test_store_failed_rav(pgpool: PgPool) { + let args = + create_sender_allocation_args(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL, None) + .await; + let _state = SenderAllocationState::new(args); + + // state.store_failed_rav() + + // check database if failed rav is stored + } + + #[sqlx::test(migrations = "../migrations")] + fn test_store_invalid_receipts(pgpool: PgPool) { + let args = + create_sender_allocation_args(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL, None) + .await; + let _state = SenderAllocationState::new(args); + // verify if invalid receipts are stored correctly } - #[test] - fn test_mark_rav_final() { + #[sqlx::test(migrations = "../migrations")] + fn test_mark_rav_final(pgpool: PgPool) { + let signed_rav = create_rav(*ALLOCATION_ID_0, SIGNER.0.clone(), 4, 10); + store_rav(&pgpool, signed_rav, SENDER.1).await.unwrap(); + + let args = + create_sender_allocation_args(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL, None) + .await; + let state = SenderAllocationState::new(args); + // mark rav as final + let result = state.mark_rav_final().await; // check if database contains final rav + assert!(result.is_ok()); } } diff --git a/tap-agent/src/agent/unaggregated_receipts.rs b/tap-agent/src/agent/unaggregated_receipts.rs index 0511152c7..96a410abf 100644 --- a/tap-agent/src/agent/unaggregated_receipts.rs +++ b/tap-agent/src/agent/unaggregated_receipts.rs @@ -1,7 +1,7 @@ // Copyright 2023-, GraphOps and Semiotic Labs. // SPDX-License-Identifier: Apache-2.0 -#[derive(Default, Debug, Clone)] +#[derive(Default, Debug, Clone, Eq, PartialEq)] pub struct UnaggregatedReceipts { pub value: u128, /// The ID of the last receipt value added to the unaggregated fees value. diff --git a/tap-agent/src/tap/context.rs b/tap-agent/src/tap/context.rs index 2cf9c0db4..716330d7a 100644 --- a/tap-agent/src/tap/context.rs +++ b/tap-agent/src/tap/context.rs @@ -21,7 +21,6 @@ pub struct TapAgentContext { allocation_id: Address, sender: Address, escrow_accounts: Eventual, - // sender_pending_fees: Arc>>, escrow_adapter: EscrowAdapter, } @@ -38,7 +37,6 @@ impl TapAgentContext { allocation_id, sender, escrow_accounts, - // sender_pending_fees: Arc::new(RwLock::new(HashMap::new())), escrow_adapter, } } diff --git a/tap-agent/src/tap/context/rav.rs b/tap-agent/src/tap/context/rav.rs index dabccb026..c3284ad7a 100644 --- a/tap-agent/src/tap/context/rav.rs +++ b/tap-agent/src/tap/context/rav.rs @@ -127,36 +127,15 @@ impl RAVStore for TapAgentContext { #[cfg(test)] mod test { - use ethers_signers::LocalWallet; use eventuals::Eventual; use sqlx::PgPool; - use tap_core::signed_message::EIP712SignedMessage; use super::*; use crate::tap::{ escrow_adapter::EscrowAdapter, - test_utils::{ALLOCATION_ID_0, SENDER, SIGNER, TAP_EIP712_DOMAIN_SEPARATOR}, + test_utils::{create_rav, ALLOCATION_ID_0, SENDER, SIGNER}, }; - /// Fixture to generate a RAV using the wallet from `keys()` - fn create_rav( - allocation_id: Address, - signer_wallet: LocalWallet, - timestamp_ns: u64, - value_aggregate: u128, - ) -> SignedRAV { - EIP712SignedMessage::new( - &TAP_EIP712_DOMAIN_SEPARATOR, - ReceiptAggregateVoucher { - allocationId: allocation_id, - timestampNs: timestamp_ns, - valueAggregate: value_aggregate, - }, - &signer_wallet, - ) - .unwrap() - } - #[sqlx::test(migrations = "../migrations")] async fn update_and_retrieve_rav(pool: PgPool) { let timestamp_ns = u64::MAX - 10; diff --git a/tap-agent/src/tap/context/receipt.rs b/tap-agent/src/tap/context/receipt.rs index 454246093..179aeff48 100644 --- a/tap-agent/src/tap/context/receipt.rs +++ b/tap-agent/src/tap/context/receipt.rs @@ -187,17 +187,18 @@ mod test { use super::*; use crate::tap::{ escrow_adapter::EscrowAdapter, - test_utils::{wallet, ALLOCATION_ID_0, SENDER, SIGNER, TAP_EIP712_DOMAIN_SEPARATOR}, + test_utils::{ + create_received_receipt, store_receipt, wallet, ALLOCATION_ID_0, SENDER, SIGNER, + TAP_EIP712_DOMAIN_SEPARATOR, + }, }; use anyhow::Result; - use bigdecimal::num_bigint::BigInt; use ethers_signers::LocalWallet; use eventuals::Eventual; use indexer_common::escrow_accounts::EscrowAccounts; use lazy_static::lazy_static; use sqlx::PgPool; use std::collections::HashMap; - use tap_core::signed_message::EIP712SignedMessage; lazy_static! { pub static ref SENDER_IRRELEVANT: (LocalWallet, Address) = wallet(1); @@ -205,56 +206,6 @@ mod test { Address::from_str("0xbcdebcdebcdebcdebcdebcdebcdebcdebcdebcde").unwrap(); } - /// Fixture to generate a signed receipt using the wallet from `keys()` and the - /// given `query_id` and `value` - fn create_received_receipt( - allocation_id: &Address, - signer_wallet: &LocalWallet, - nonce: u64, - timestamp_ns: u64, - value: u128, - ) -> ReceiptWithState { - let receipt = EIP712SignedMessage::new( - &TAP_EIP712_DOMAIN_SEPARATOR, - Receipt { - allocation_id: *allocation_id, - nonce, - timestamp_ns, - value, - }, - signer_wallet, - ) - .unwrap(); - ReceiptWithState::new(receipt) - } - - pub async fn store_receipt(pgpool: &PgPool, signed_receipt: &SignedReceipt) -> Result { - let encoded_signature = signed_receipt.signature.to_vec(); - - let record = sqlx::query!( - r#" - INSERT INTO scalar_tap_receipts (signer_address, signature, allocation_id, timestamp_ns, nonce, value) - VALUES ($1, $2, $3, $4, $5, $6) - RETURNING id - "#, - signed_receipt - .recover_signer(&TAP_EIP712_DOMAIN_SEPARATOR) - .unwrap() - .encode_hex::(), - encoded_signature, - signed_receipt.message.allocation_id.encode_hex::(), - BigDecimal::from(signed_receipt.message.timestamp_ns), - BigDecimal::from(signed_receipt.message.nonce), - BigDecimal::from(BigInt::from(signed_receipt.message.value)), - ) - .fetch_one(pgpool) - .await?; - - // id is BIGSERIAL, so it should be safe to cast to u64. - let id: u64 = record.id.try_into()?; - Ok(id) - } - /// Insert a single receipt and retrieve it from the database using the adapter. /// The point here it to test the deserialization of large numbers. #[sqlx::test(migrations = "../migrations")] diff --git a/tap-agent/src/tap/mod.rs b/tap-agent/src/tap/mod.rs index 25998ebb6..3802cf8b7 100644 --- a/tap-agent/src/tap/mod.rs +++ b/tap-agent/src/tap/mod.rs @@ -11,7 +11,7 @@ pub mod context; pub mod escrow_adapter; #[cfg(test)] -mod test_utils; +pub mod test_utils; pub async fn signers_trimmed( escrow_accounts: &Eventual, diff --git a/tap-agent/src/tap/test_utils.rs b/tap-agent/src/tap/test_utils.rs index 65af769fd..9c2a82066 100644 --- a/tap-agent/src/tap/test_utils.rs +++ b/tap-agent/src/tap/test_utils.rs @@ -3,9 +3,20 @@ use std::str::FromStr; +use alloy_primitives::hex::ToHex; +use bigdecimal::num_bigint::BigInt; + +use sqlx::types::BigDecimal; + use alloy_sol_types::{eip712_domain, Eip712Domain}; use ethers_signers::{coins_bip39::English, LocalWallet, MnemonicBuilder, Signer}; use lazy_static::lazy_static; +use sqlx::PgPool; +use tap_core::{ + rav::{ReceiptAggregateVoucher, SignedRAV}, + receipt::{Checking, Receipt, ReceiptWithState, SignedReceipt}, + signed_message::EIP712SignedMessage, +}; use thegraph::types::Address; lazy_static! { @@ -13,6 +24,7 @@ lazy_static! { Address::from_str("0xabababababababababababababababababababab").unwrap(); pub static ref SENDER: (LocalWallet, Address) = wallet(0); pub static ref SIGNER: (LocalWallet, Address) = wallet(2); + pub static ref INDEXER: (LocalWallet, Address) = wallet(3); pub static ref TAP_EIP712_DOMAIN_SEPARATOR: Eip712Domain = eip712_domain! { name: "TAP", version: "1", @@ -21,6 +33,75 @@ lazy_static! { }; } +/// Fixture to generate a RAV using the wallet from `keys()` +pub fn create_rav( + allocation_id: Address, + signer_wallet: LocalWallet, + timestamp_ns: u64, + value_aggregate: u128, +) -> SignedRAV { + EIP712SignedMessage::new( + &TAP_EIP712_DOMAIN_SEPARATOR, + ReceiptAggregateVoucher { + allocationId: allocation_id, + timestampNs: timestamp_ns, + valueAggregate: value_aggregate, + }, + &signer_wallet, + ) + .unwrap() +} + +/// Fixture to generate a signed receipt using the wallet from `keys()` and the +/// given `query_id` and `value` +pub fn create_received_receipt( + allocation_id: &Address, + signer_wallet: &LocalWallet, + nonce: u64, + timestamp_ns: u64, + value: u128, +) -> ReceiptWithState { + let receipt = EIP712SignedMessage::new( + &TAP_EIP712_DOMAIN_SEPARATOR, + Receipt { + allocation_id: *allocation_id, + nonce, + timestamp_ns, + value, + }, + signer_wallet, + ) + .unwrap(); + ReceiptWithState::new(receipt) +} + +pub async fn store_receipt(pgpool: &PgPool, signed_receipt: &SignedReceipt) -> anyhow::Result { + let encoded_signature = signed_receipt.signature.to_vec(); + + let record = sqlx::query!( + r#" + INSERT INTO scalar_tap_receipts (signer_address, signature, allocation_id, timestamp_ns, nonce, value) + VALUES ($1, $2, $3, $4, $5, $6) + RETURNING id + "#, + signed_receipt + .recover_signer(&TAP_EIP712_DOMAIN_SEPARATOR) + .unwrap() + .encode_hex::(), + encoded_signature, + signed_receipt.message.allocation_id.encode_hex::(), + BigDecimal::from(signed_receipt.message.timestamp_ns), + BigDecimal::from(signed_receipt.message.nonce), + BigDecimal::from(BigInt::from(signed_receipt.message.value)), + ) + .fetch_one(pgpool) + .await?; + + // id is BIGSERIAL, so it should be safe to cast to u64. + let id: u64 = record.id.try_into()?; + Ok(id) +} + /// Fixture to generate a wallet and address pub fn wallet(index: u32) -> (LocalWallet, Address) { let wallet: LocalWallet = MnemonicBuilder::::default() @@ -32,3 +113,27 @@ pub fn wallet(index: u32) -> (LocalWallet, Address) { let address = wallet.address(); (wallet, Address::from_slice(address.as_bytes())) } + +pub async fn store_rav( + pgpool: &PgPool, + signed_rav: SignedRAV, + sender: Address, +) -> anyhow::Result<()> { + let signature_bytes = signed_rav.signature.to_vec(); + + let _fut = sqlx::query!( + r#" + INSERT INTO scalar_tap_ravs (sender_address, signature, allocation_id, timestamp_ns, value_aggregate) + VALUES ($1, $2, $3, $4, $5) + "#, + sender.encode_hex::(), + signature_bytes, + signed_rav.message.allocationId.encode_hex::(), + BigDecimal::from(signed_rav.message.timestampNs), + BigDecimal::from(BigInt::from(signed_rav.message.valueAggregate)), + ) + .execute(pgpool) + .await?; + + Ok(()) +} diff --git a/tap-agent/tests/sender_allocation_tests.rs b/tap-agent/tests/sender_allocation_tests.rs deleted file mode 100644 index 6a1b0bc5d..000000000 --- a/tap-agent/tests/sender_allocation_tests.rs +++ /dev/null @@ -1,238 +0,0 @@ -// Copyright 2023-, GraphOps and Semiotic Labs. -// SPDX-License-Identifier: Apache-2.0 - -use std::collections::HashMap; - -use eventuals::Eventual; -use indexer_common::{ - escrow_accounts::EscrowAccounts, - subgraph_client::{DeploymentDetails, SubgraphClient}, -}; -use indexer_tap_agent::{ - agent::{ - sender_account::SenderAccountMessage, - sender_allocation::{SenderAllocation, SenderAllocationArgs, SenderAllocationMessage}, - }, - config, - tap::escrow_adapter::EscrowAdapter, -}; -use ractor::{call, Actor, ActorProcessingErr, ActorRef}; -use serde_json::json; -use sqlx::PgPool; -use tap_aggregator::server::run_server; - -use wiremock::{ - matchers::{body_string_contains, method}, - Mock, MockServer, ResponseTemplate, -}; - -mod test_utils; - -struct MockSenderAccount; - -#[async_trait::async_trait] -impl Actor for MockSenderAccount { - type Msg = SenderAccountMessage; - type State = (); - type Arguments = (); - - async fn pre_start( - &self, - _myself: ActorRef, - _allocation_ids: Self::Arguments, - ) -> std::result::Result { - Ok(()) - } -} - -use test_utils::{ - create_rav, create_received_receipt, store_rav, store_receipt, ALLOCATION_ID_0, INDEXER, - SENDER, SIGNER, TAP_EIP712_DOMAIN_SEPARATOR, -}; - -const DUMMY_URL: &str = "http://localhost:1234"; - -async fn create_sender_allocation( - pgpool: PgPool, - sender_aggregator_endpoint: String, - escrow_subgraph_endpoint: &str, -) -> ActorRef { - let config = Box::leak(Box::new(config::Cli { - config: None, - ethereum: config::Ethereum { - indexer_address: INDEXER.1, - }, - tap: config::Tap { - rav_request_trigger_value: 100, - rav_request_timestamp_buffer_ms: 1, - rav_request_timeout_secs: 5, - ..Default::default() - }, - ..Default::default() - })); - - let escrow_subgraph = Box::leak(Box::new(SubgraphClient::new( - reqwest::Client::new(), - None, - DeploymentDetails::for_query_url(escrow_subgraph_endpoint).unwrap(), - ))); - - let escrow_accounts_eventual = Eventual::from_value(EscrowAccounts::new( - HashMap::from([(SENDER.1, 1000.into())]), - HashMap::from([(SENDER.1, vec![SIGNER.1])]), - )); - - let escrow_adapter = EscrowAdapter::new(escrow_accounts_eventual.clone(), SENDER.1); - - let (sender_account_ref, _join_handle) = MockSenderAccount::spawn(None, MockSenderAccount, ()) - .await - .unwrap(); - - let args = SenderAllocationArgs { - config, - pgpool: pgpool.clone(), - allocation_id: *ALLOCATION_ID_0, - sender: SENDER.1, - escrow_accounts: escrow_accounts_eventual, - escrow_subgraph, - escrow_adapter, - domain_separator: TAP_EIP712_DOMAIN_SEPARATOR.clone(), - sender_aggregator_endpoint, - sender_account_ref, - }; - - let (allocation_ref, _join_handle) = SenderAllocation::spawn(None, SenderAllocation, args) - .await - .unwrap(); - - allocation_ref -} - -/// Test that the sender_allocation correctly updates the unaggregated fees from the -/// database when there is no RAV in the database. -/// -/// The sender_allocation should consider all receipts found for the allocation and -/// sender. -#[sqlx::test(migrations = "../migrations")] -async fn test_update_unaggregated_fees_no_rav(pgpool: PgPool) { - // Add receipts to the database. - for i in 1..10 { - let receipt = create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i, i.into()).await; - store_receipt(&pgpool, receipt.signed_receipt()) - .await - .unwrap(); - } - - let sender_allocation = - create_sender_allocation(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL).await; - - // Get total_unaggregated_fees - let total_unaggregated_fees = call!( - sender_allocation, - SenderAllocationMessage::GetUnaggregatedReceipts - ) - .unwrap(); - - // Check that the unaggregated fees are correct. - assert_eq!(total_unaggregated_fees.value, 45u128); -} - -/// Test that the sender_allocation correctly updates the unaggregated fees from the -/// database when there is a RAV in the database as well as receipts which timestamp are lesser -/// and greater than the RAV's timestamp. -/// -/// The sender_allocation should only consider receipts with a timestamp greater -/// than the RAV's timestamp. -#[sqlx::test(migrations = "../migrations")] -async fn test_update_unaggregated_fees_with_rav(pgpool: PgPool) { - // Add the RAV to the database. - // This RAV has timestamp 4. The sender_allocation should only consider receipts - // with a timestamp greater than 4. - let signed_rav = create_rav(*ALLOCATION_ID_0, SIGNER.0.clone(), 4, 10).await; - store_rav(&pgpool, signed_rav, SENDER.1).await.unwrap(); - - // Add receipts to the database. - for i in 1..10 { - let receipt = create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i, i.into()).await; - store_receipt(&pgpool, receipt.signed_receipt()) - .await - .unwrap(); - } - - let sender_allocation = - create_sender_allocation(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL).await; - - // Get total_unaggregated_fees - let total_unaggregated_fees = call!( - sender_allocation, - SenderAllocationMessage::GetUnaggregatedReceipts - ) - .unwrap(); - - // Check that the unaggregated fees are correct. - assert_eq!(total_unaggregated_fees.value, 35u128); -} - -#[sqlx::test(migrations = "../migrations")] -async fn test_rav_requester_manual(pgpool: PgPool) { - // Start a TAP aggregator server. - let (handle, aggregator_endpoint) = run_server( - 0, - SIGNER.0.clone(), - vec![SIGNER.1].into_iter().collect(), - TAP_EIP712_DOMAIN_SEPARATOR.clone(), - 100 * 1024, - 100 * 1024, - 1, - ) - .await - .unwrap(); - - // Start a mock graphql server using wiremock - let mock_server = MockServer::start().await; - - // Mock result for TAP redeem txs for (allocation, sender) pair. - mock_server - .register( - Mock::given(method("POST")) - .and(body_string_contains("transactions")) - .respond_with( - ResponseTemplate::new(200) - .set_body_json(json!({ "data": { "transactions": []}})), - ), - ) - .await; - - // Add receipts to the database. - for i in 0..10 { - let receipt = - create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i + 1, i.into()).await; - store_receipt(&pgpool, receipt.signed_receipt()) - .await - .unwrap(); - } - - // Create a sender_allocation. - let sender_allocation = create_sender_allocation( - pgpool.clone(), - "http://".to_owned() + &aggregator_endpoint.to_string(), - &mock_server.uri(), - ) - .await; - - // Trigger a RAV request manually. - - // Get total_unaggregated_fees - let total_unaggregated_fees = call!( - sender_allocation, - SenderAllocationMessage::TriggerRAVRequest - ) - .unwrap(); - - // Check that the unaggregated fees are correct. - assert_eq!(total_unaggregated_fees.value, 0u128); - - // Stop the TAP aggregator server. - handle.stop().unwrap(); - handle.stopped().await; -} From df1432688e3b60eef301c1e023bb6ef7f09d7ea9 Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Tue, 2 Apr 2024 22:23:35 -0300 Subject: [PATCH 19/41] test: finish sender_allocation tests Signed-off-by: Gustavo Inacio --- Cargo.lock | 37 +-- tap-agent/Cargo.toml | 1 + tap-agent/src/agent/sender_allocation.rs | 320 ++++++++++++++++++----- 3 files changed, 273 insertions(+), 85 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6584cecf5..9ed6b2125 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2419,9 +2419,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" dependencies = [ "futures-channel", "futures-core", @@ -2434,9 +2434,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -2444,15 +2444,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" dependencies = [ "futures-core", "futures-task", @@ -2472,9 +2472,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-lite" @@ -2503,9 +2503,9 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", @@ -2514,15 +2514,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-timer" @@ -2536,9 +2536,9 @@ dependencies = [ [[package]] name = "futures-util" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-channel", "futures-core", @@ -3162,6 +3162,7 @@ dependencies = [ "ethers", "ethers-signers", "eventuals", + "futures", "graphql-http", "indexer-common", "jsonrpsee 0.20.2", diff --git a/tap-agent/Cargo.toml b/tap-agent/Cargo.toml index 3b5db4cd6..d946ef84a 100644 --- a/tap-agent/Cargo.toml +++ b/tap-agent/Cargo.toml @@ -56,3 +56,4 @@ ractor = "0.9.7" ethers-signers = "2.0.8" tempfile = "3.8.0" wiremock = "0.5.19" +futures = "0.3.30" diff --git a/tap-agent/src/agent/sender_allocation.rs b/tap-agent/src/agent/sender_allocation.rs index c16081b13..10982fc61 100644 --- a/tap-agent/src/agent/sender_allocation.rs +++ b/tap-agent/src/agent/sender_allocation.rs @@ -143,34 +143,42 @@ impl Actor for SenderAllocation { ))?; } } + // we use a blocking call here to ensure that only one RAV request is running at a time. SenderAllocationMessage::TriggerRAVRequest(reply) => { - state.rav_requester_single().await.map_err(|e| { - anyhow! { - "Error while requesting RAV for sender {} and allocation {}: {}", - state.sender, - state.allocation_id, - e - } - })?; - state.unaggregated_fees = state.calculate_unaggregated_fee().await?; + if state.unaggregated_fees.value > 0 { + state.rav_requester_single().await.map_err(|e| { + anyhow! { + "Error while requesting RAV for sender {} and allocation {}: {}", + state.sender, + state.allocation_id, + e + } + })?; + state.unaggregated_fees = state.calculate_unaggregated_fee().await?; + } if !reply.is_closed() { let _ = reply.send(state.unaggregated_fees.clone()); } } SenderAllocationMessage::CloseAllocation => { - state.rav_requester_single().await.inspect_err(|e| { - error!( - "Error while requesting RAV for sender {} and allocation {}: {}", - state.sender, state.allocation_id, e - ); - })?; + // Request a RAV and mark the allocation as final. + + if state.unaggregated_fees.value > 0 { + state.rav_requester_single().await.inspect_err(|e| { + error!( + "Error while requesting RAV for sender {} and allocation {}: {}", + state.sender, state.allocation_id, e + ); + })?; + } state.mark_rav_final().await.inspect_err(|e| { error!( "Error while marking allocation {} as final for sender {}: {}", state.allocation_id, state.sender, e ); })?; + // stop and trigger post_stop myself.stop(None); } SenderAllocationMessage::GetUnaggregatedReceipts(reply) => { @@ -400,14 +408,23 @@ impl SenderAllocationState { ) .execute(&self.pgpool) .await?; - if updated_rows.rows_affected() != 1 { - anyhow::bail!( + + match updated_rows.rows_affected() { + // in case no rav was marked as final + 0 => { + warn!( + "No RAVs were updated as final for allocation {} and sender {}.", + self.allocation_id, self.sender + ); + Ok(()) + } + 1 => Ok(()), + _ => anyhow::bail!( "Expected exactly one row to be updated in the latest RAVs table, \ but {} were updated.", updated_rows.rows_affected() - ); - }; - Ok(()) + ), + } } async fn store_invalid_receipts(&self, receipts: &[ReceiptWithState]) -> Result<()> { @@ -483,25 +500,9 @@ impl SenderAllocationState { #[cfg(test)] mod tests { - use std::{ - collections::HashMap, - sync::{Arc, Mutex}, - }; - - use eventuals::Eventual; - use indexer_common::{ - escrow_accounts::EscrowAccounts, - subgraph_client::{DeploymentDetails, SubgraphClient}, - }; - use ractor::{call, cast, concurrency::JoinHandle, Actor, ActorProcessingErr, ActorRef}; - use serde_json::json; - use sqlx::PgPool; - use tap_aggregator::server::run_server; - use wiremock::{ - matchers::{body_string_contains, method}, - Mock, MockServer, ResponseTemplate, + use super::{ + SenderAllocation, SenderAllocationArgs, SenderAllocationMessage, SenderAllocationState, }; - use crate::{ agent::{ sender_account::SenderAccountMessage, sender_accounts_manager::NewReceiptNotification, @@ -516,15 +517,35 @@ mod tests { }, }, }; - - use super::{ - SenderAllocation, SenderAllocationArgs, SenderAllocationMessage, SenderAllocationState, + use eventuals::Eventual; + use futures::future::join_all; + use indexer_common::{ + escrow_accounts::EscrowAccounts, + subgraph_client::{DeploymentDetails, SubgraphClient}, + }; + use ractor::{ + call, cast, concurrency::JoinHandle, Actor, ActorProcessingErr, ActorRef, ActorStatus, + }; + use serde_json::json; + use sqlx::PgPool; + use std::{ + collections::HashMap, + sync::{Arc, Mutex}, + }; + use tap_aggregator::{jsonrpsee_helpers::JsonRpcResponse, server::run_server}; + use tap_core::receipt::{ + checks::{Check, Checks}, + Checking, ReceiptWithState, + }; + use wiremock::{ + matchers::{body_string_contains, method}, + Mock, MockServer, Respond, ResponseTemplate, }; const DUMMY_URL: &str = "http://localhost:1234"; struct MockSenderAccount { - last_message_emitted: Arc>>, + last_message_emitted: Arc>>, } #[async_trait::async_trait] @@ -547,17 +568,17 @@ mod tests { message: Self::Msg, _state: &mut Self::State, ) -> std::result::Result<(), ActorProcessingErr> { - *self.last_message_emitted.lock().unwrap() = Some(message); + self.last_message_emitted.lock().unwrap().push(message); Ok(()) } } async fn create_mock_sender_account() -> ( - Arc>>, + Arc>>, ActorRef, JoinHandle<()>, ) { - let last_message_emitted = Arc::new(Mutex::new(None)); + let last_message_emitted = Arc::new(Mutex::new(vec![])); let (sender_account, join_handle) = MockSenderAccount::spawn( None, @@ -646,16 +667,23 @@ mod tests { #[sqlx::test(migrations = "../migrations")] async fn should_update_unaggregated_fees_on_start(pgpool: PgPool) { + let (last_message_emitted, sender_account, _join_handle) = + create_mock_sender_account().await; // Add receipts to the database. - for i in 1..10 { + for i in 1..=10 { let receipt = create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i, i.into()); store_receipt(&pgpool, receipt.signed_receipt()) .await .unwrap(); } - let sender_allocation = - create_sender_allocation(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL, None).await; + let sender_allocation = create_sender_allocation( + pgpool.clone(), + DUMMY_URL.to_string(), + DUMMY_URL, + Some(sender_account), + ) + .await; // Get total_unaggregated_fees let total_unaggregated_fees = call!( @@ -664,8 +692,20 @@ mod tests { ) .unwrap(); + // Should emit a message to the sender account with the unaggregated fees. + let expected_message = SenderAccountMessage::UpdateReceiptFees( + *ALLOCATION_ID_0, + UnaggregatedReceipts { + last_id: 10, + value: 55u128, + }, + ); + let last_message_emitted = last_message_emitted.lock().unwrap(); + assert_eq!(last_message_emitted.len(), 1); + assert_eq!(last_message_emitted.last(), Some(&expected_message)); + // Check that the unaggregated fees are correct. - assert_eq!(total_unaggregated_fees.value, 45u128); + assert_eq!(total_unaggregated_fees.value, 55u128); } #[sqlx::test(migrations = "../migrations")] @@ -716,10 +756,9 @@ mod tests { value: 20, }, ); - assert_eq!( - *last_message_emitted.lock().unwrap(), - Some(expected_message) - ); + let last_message_emitted = last_message_emitted.lock().unwrap(); + assert_eq!(last_message_emitted.len(), 2); + assert_eq!(last_message_emitted.last(), Some(&expected_message)); } #[sqlx::test(migrations = "../migrations")] @@ -769,9 +808,7 @@ mod tests { ) .await; - // Trigger a RAV request manually. - - // Get total_unaggregated_fees + // Trigger a RAV request manually and wait for updated fees. let total_unaggregated_fees = call!( sender_allocation, SenderAllocationMessage::TriggerRAVRequest @@ -786,19 +823,136 @@ mod tests { handle.stopped().await; } - #[test] - fn test_close_allocation() { + #[sqlx::test(migrations = "../migrations")] + async fn test_close_allocation_no_pending_fees(pgpool: PgPool) { + let (last_message_emitted, sender_account, _join_handle) = + create_mock_sender_account().await; + + // create allocation + let sender_allocation = create_sender_allocation( + pgpool.clone(), + DUMMY_URL.to_string(), + DUMMY_URL, + Some(sender_account), + ) + .await; + + cast!(sender_allocation, SenderAllocationMessage::CloseAllocation).unwrap(); + + // should trigger rav request + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + + // check if the actor is actually stopped + assert_eq!(sender_allocation.get_status(), ActorStatus::Stopped); + + // check if message is sent to sender account + assert_eq!( + last_message_emitted.lock().unwrap().last(), + Some(&SenderAccountMessage::UpdateReceiptFees( + *ALLOCATION_ID_0, + UnaggregatedReceipts::default() + )) + ); + } + + #[sqlx::test(migrations = "../migrations")] + async fn test_close_allocation_with_pending_fees(pgpool: PgPool) { + struct Response { + data: Arc, + } + + impl Respond for Response { + fn respond(&self, _request: &wiremock::Request) -> wiremock::ResponseTemplate { + self.data.notify_one(); + + let mock_rav = create_rav(*ALLOCATION_ID_0, SIGNER.0.clone(), 10, 45); + + let json_response = JsonRpcResponse { + data: mock_rav, + warnings: None, + }; + + ResponseTemplate::new(200).set_body_json(json! ( + { + "id": 0, + "jsonrpc": "2.0", + "result": json_response + } + )) + } + } + + let await_trigger = Arc::new(tokio::sync::Notify::new()); + // Start a TAP aggregator server. + let aggregator_server = MockServer::start().await; + + aggregator_server + .register( + Mock::given(method("POST")) + .and(body_string_contains("aggregate_receipts")) + .respond_with(Response { + data: await_trigger.clone(), + }), + ) + .await; + + // Start a mock graphql server using wiremock + let mock_server = MockServer::start().await; + + // Mock result for TAP redeem txs for (allocation, sender) pair. + mock_server + .register( + Mock::given(method("POST")) + .and(body_string_contains("transactions")) + .respond_with( + ResponseTemplate::new(200) + .set_body_json(json!({ "data": { "transactions": []}})), + ), + ) + .await; + + // Add receipts to the database. + for i in 0..10 { + let receipt = create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i + 1, i.into()); + store_receipt(&pgpool, receipt.signed_receipt()) + .await + .unwrap(); + } + + let (last_message_emitted, sender_account, _join_handle) = + create_mock_sender_account().await; + // create allocation + let sender_allocation = create_sender_allocation( + pgpool.clone(), + aggregator_server.uri(), + &mock_server.uri(), + Some(sender_account), + ) + .await; - // populate db + cast!(sender_allocation, SenderAllocationMessage::CloseAllocation).unwrap(); // should trigger rav request + await_trigger.notified().await; + tokio::time::sleep(std::time::Duration::from_millis(10)).await; - // should mark rav final + // check if rav request is made + assert!(aggregator_server.received_requests().await.is_some()); - // should stop actor + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + + // check if the actor is actually stopped + assert_eq!(sender_allocation.get_status(), ActorStatus::Stopped); // check if message is sent to sender account + assert_eq!( + last_message_emitted.lock().unwrap().last(), + Some(&SenderAccountMessage::UpdateReceiptFees( + *ALLOCATION_ID_0, + UnaggregatedReceipts::default() + )) + ); } #[sqlx::test(migrations = "../migrations")] @@ -861,21 +1015,53 @@ mod tests { let args = create_sender_allocation_args(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL, None) .await; - let _state = SenderAllocationState::new(args); + let state = SenderAllocationState::new(args); - // state.store_failed_rav() + let signed_rav = create_rav(*ALLOCATION_ID_0, SIGNER.0.clone(), 4, 10); - // check database if failed rav is stored + // just unit test if it is working + let result = state + .store_failed_rav(&signed_rav.message, &signed_rav, "test") + .await; + + assert!(result.is_ok()); } #[sqlx::test(migrations = "../migrations")] fn test_store_invalid_receipts(pgpool: PgPool) { + struct FailingCheck; + + #[async_trait::async_trait] + impl Check for FailingCheck { + async fn check(&self, _receipt: &ReceiptWithState) -> anyhow::Result<()> { + Err(anyhow::anyhow!("Failing check")) + } + } + let args = create_sender_allocation_args(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL, None) .await; - let _state = SenderAllocationState::new(args); + let state = SenderAllocationState::new(args); - // verify if invalid receipts are stored correctly + let checks = Checks::new(vec![Arc::new(FailingCheck)]); + + // create some checks + let checking_receipts = vec![ + create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, 1, 1, 1u128), + create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, 2, 2, 2u128), + ]; + // make sure to fail them + let failing_receipts = checking_receipts + .into_iter() + .map(|receipt| async { receipt.finalize_receipt_checks(&checks).await.unwrap_err() }) + .collect::>(); + let failing_receipts = join_all(failing_receipts).await; + + // store the failing receipts + let result = state.store_invalid_receipts(&failing_receipts).await; + + // we just store a few and make sure it doesn't fail + assert!(result.is_ok()); } #[sqlx::test(migrations = "../migrations")] @@ -891,7 +1077,7 @@ mod tests { // mark rav as final let result = state.mark_rav_final().await; - // check if database contains final rav + // check if it fails assert!(result.is_ok()); } } From 887fab0339590daefa8199006c4934d6c1d13985 Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Mon, 8 Apr 2024 23:45:03 -0300 Subject: [PATCH 20/41] test: add tests to sender accounts manager Signed-off-by: Gustavo Inacio --- tap-agent/src/agent.rs | 1 + tap-agent/src/agent/sender_account.rs | 21 +- .../src/agent/sender_accounts_manager.rs | 261 +++++++++++++++++- tap-agent/src/agent/sender_allocation.rs | 8 +- tap-agent/src/tap/test_utils.rs | 3 + tap-agent/tests/sender_manager_tests.rs | 152 ---------- 6 files changed, 273 insertions(+), 173 deletions(-) delete mode 100644 tap-agent/tests/sender_manager_tests.rs diff --git a/tap-agent/src/agent.rs b/tap-agent/src/agent.rs index 5956749eb..40cbca000 100644 --- a/tap-agent/src/agent.rs +++ b/tap-agent/src/agent.rs @@ -111,6 +111,7 @@ pub async fn start_agent() -> ActorRef { escrow_accounts, escrow_subgraph, sender_aggregator_endpoints, + prefix: None, }; let (manager, _) = SenderAccountsManager::spawn(None, SenderAccountsManager, args) diff --git a/tap-agent/src/agent/sender_account.rs b/tap-agent/src/agent/sender_account.rs index 202b1250b..a2b2eaead 100644 --- a/tap-agent/src/agent/sender_account.rs +++ b/tap-agent/src/agent/sender_account.rs @@ -7,7 +7,7 @@ use alloy_sol_types::Eip712Domain; use anyhow::Result; use eventuals::{Eventual, EventualExt, PipeHandle}; use indexer_common::{escrow_accounts::EscrowAccounts, prelude::SubgraphClient}; -use ractor::{call, Actor, ActorProcessingErr, ActorRef}; +use ractor::{call, Actor, ActorProcessingErr, ActorRef, SupervisionEvent}; use sqlx::PgPool; use thegraph::types::Address; use tracing::error; @@ -162,6 +162,21 @@ impl Actor for SenderAccount { }) } + async fn handle_supervisor_evt( + &self, + _myself: ActorRef, + message: SupervisionEvent, + _state: &mut Self::State, + ) -> std::result::Result<(), ActorProcessingErr> { + match message { + SupervisionEvent::ActorTerminated(_, _, _) | SupervisionEvent::ActorPanicked(_, _) => { + // what to do in case of termination or panic? + } + _ => {} + } + Ok(()) + } + async fn handle( &self, myself: ActorRef, @@ -169,7 +184,9 @@ impl Actor for SenderAccount { state: &mut Self::State, ) -> std::result::Result<(), ActorProcessingErr> { match message { - SenderAccountMessage::RemoveSenderAccount => myself.stop(None), + SenderAccountMessage::RemoveSenderAccount => { + myself.stop(Some("Received Remove Sender Account message".into())) + } SenderAccountMessage::UpdateReceiptFees(allocation_id, unaggregated_fees) => { let tracker = &mut state.allocation_id_tracker; tracker.add_or_update(allocation_id, unaggregated_fees.value); diff --git a/tap-agent/src/agent/sender_accounts_manager.rs b/tap-agent/src/agent/sender_accounts_manager.rs index ec992c10a..e3d1e2429 100644 --- a/tap-agent/src/agent/sender_accounts_manager.rs +++ b/tap-agent/src/agent/sender_accounts_manager.rs @@ -11,10 +11,11 @@ use anyhow::Result; use eventuals::{Eventual, EventualExt, PipeHandle}; use indexer_common::escrow_accounts::EscrowAccounts; use indexer_common::prelude::{Allocation, SubgraphClient}; -use ractor::{Actor, ActorProcessingErr, ActorRef}; +use ractor::{Actor, ActorProcessingErr, ActorRef, SupervisionEvent}; use serde::Deserialize; use sqlx::{postgres::PgListener, PgPool}; use thegraph::types::Address; +use tokio::select; use tracing::{error, warn}; use super::sender_account::{SenderAccount, SenderAccountArgs, SenderAccountMessage}; @@ -45,6 +46,8 @@ pub struct SenderAccountsManagerArgs { pub escrow_accounts: Eventual, pub escrow_subgraph: &'static SubgraphClient, pub sender_aggregator_endpoints: HashMap, + + pub prefix: Option, } pub struct State { @@ -59,6 +62,7 @@ pub struct State { escrow_accounts: Eventual, escrow_subgraph: &'static SubgraphClient, sender_aggregator_endpoints: HashMap, + prefix: Option, } #[async_trait::async_trait] @@ -78,6 +82,7 @@ impl Actor for SenderAccountsManager { escrow_accounts, escrow_subgraph, sender_aggregator_endpoints, + prefix, }: Self::Arguments, ) -> std::result::Result { let indexer_allocations = indexer_allocations.map(|allocations| async move { @@ -121,8 +126,15 @@ impl Actor for SenderAccountsManager { escrow_accounts, escrow_subgraph, sender_aggregator_endpoints, + prefix, + }; + let sender_allocation = select! { + sender_allocation = state.get_pending_sender_allocation_id() => sender_allocation, + _ = tokio::time::sleep(std::time::Duration::from_secs(30)) => { + panic!("Timeout while getting pending sender allocation ids"); + } }; - let sender_allocation = state.get_pending_sender_allocation_id().await; + for (sender_id, allocation_ids) in sender_allocation { myself.cast(SenderAccountsManagerMessage::CreateSenderAccount( sender_id, @@ -143,6 +155,21 @@ impl Actor for SenderAccountsManager { Ok(()) } + async fn handle_supervisor_evt( + &self, + _myself: ActorRef, + message: SupervisionEvent, + _state: &mut Self::State, + ) -> std::result::Result<(), ActorProcessingErr> { + match message { + SupervisionEvent::ActorTerminated(_, _, _) | SupervisionEvent::ActorPanicked(_, _) => { + // what to do in case of termination or panic + } + _ => {} + } + Ok(()) + } + async fn handle( &self, myself: ActorRef, @@ -161,9 +188,9 @@ impl Actor for SenderAccountsManager { // Remove sender accounts for sender in state.sender_ids.difference(&target_senders) { - if let Some(sender_handle) = - ActorRef::::where_is(sender.to_string()) - { + if let Some(sender_handle) = ActorRef::::where_is( + state.format_sender_account(sender), + ) { sender_handle.cast(SenderAccountMessage::RemoveSenderAccount)?; } } @@ -173,7 +200,7 @@ impl Actor for SenderAccountsManager { SenderAccountsManagerMessage::CreateSenderAccount(sender_id, allocation_ids) => { let args = state.new_sender_account_args(&sender_id, allocation_ids)?; SenderAccount::spawn_linked( - Some(sender_id.to_string()), + Some(state.format_sender_account(&sender_id)), SenderAccount, args, myself.get_cell(), @@ -186,6 +213,16 @@ impl Actor for SenderAccountsManager { } impl State { + fn format_sender_account(&self, sender: &Address) -> String { + let mut sender_allocation_id = String::new(); + if let Some(prefix) = &self.prefix { + sender_allocation_id.push_str(prefix); + sender_allocation_id.push(':'); + } + sender_allocation_id.push_str(&format!("{}", sender)); + sender_allocation_id + } + async fn get_pending_sender_allocation_id(&self) -> HashMap> { let escrow_accounts_snapshot = self .escrow_accounts @@ -308,7 +345,7 @@ impl State { })? .clone(), allocation_ids, - prefix: None, + prefix: self.prefix.clone(), }) } } @@ -374,15 +411,209 @@ async fn new_receipts_watcher( #[cfg(test)] mod tests { - #[test] - fn test_create_sender_accounts_manager() {} + use super::{ + SenderAccountsManager, SenderAccountsManagerArgs, SenderAccountsManagerMessage, State, + }; + use crate::agent::sender_account::SenderAccountMessage; + use crate::config; + use crate::tap::test_utils::{ + create_rav, create_received_receipt, store_rav, store_receipt, ALLOCATION_ID_0, + ALLOCATION_ID_1, INDEXER, SENDER, SENDER_2, SIGNER, TAP_EIP712_DOMAIN_SEPARATOR, + }; + use alloy_primitives::Address; + use eventuals::{Eventual, EventualExt}; + use indexer_common::allocations::Allocation; + use indexer_common::escrow_accounts::EscrowAccounts; + use indexer_common::prelude::{DeploymentDetails, SubgraphClient}; + use ractor::concurrency::JoinHandle; + use ractor::{Actor, ActorRef}; + use sqlx::PgPool; + use std::collections::{HashMap, HashSet}; + use std::sync::atomic::AtomicU32; + + const DUMMY_URL: &str = "http://localhost:1234"; + static PREFIX_ID: AtomicU32 = AtomicU32::new(0); + + fn get_subgraph_client() -> &'static SubgraphClient { + Box::leak(Box::new(SubgraphClient::new( + reqwest::Client::new(), + None, + DeploymentDetails::for_query_url(DUMMY_URL).unwrap(), + ))) + } + + fn get_config() -> &'static config::Cli { + Box::leak(Box::new(config::Cli { + config: None, + ethereum: config::Ethereum { + indexer_address: INDEXER.1, + }, + tap: config::Tap { + rav_request_trigger_value: 100, + rav_request_timestamp_buffer_ms: 1, + ..Default::default() + }, + ..Default::default() + })) + } + + async fn create_sender_accounts_manager( + pgpool: PgPool, + ) -> ( + String, + (ActorRef, JoinHandle<()>), + ) { + let config = get_config(); + + let (mut indexer_allocations_writer, indexer_allocations_eventual) = + Eventual::>::new(); + indexer_allocations_writer.write(HashMap::new()); + let escrow_subgraph = get_subgraph_client(); + + let (mut escrow_accounts_writer, escrow_accounts_eventual) = + Eventual::::new(); + escrow_accounts_writer.write(EscrowAccounts::default()); + + let prefix = format!( + "test-{}", + PREFIX_ID.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + ); + let args = SenderAccountsManagerArgs { + config, + domain_separator: TAP_EIP712_DOMAIN_SEPARATOR.clone(), + pgpool, + indexer_allocations: indexer_allocations_eventual, + escrow_accounts: escrow_accounts_eventual, + escrow_subgraph, + sender_aggregator_endpoints: HashMap::from([ + (SENDER.1, String::from("http://localhost:8000")), + (SENDER_2.1, String::from("http://localhost:8000")), + ]), + prefix: Some(prefix.clone()), + }; + ( + prefix, + SenderAccountsManager::spawn(None, SenderAccountsManager, args) + .await + .unwrap(), + ) + } + + #[sqlx::test(migrations = "../migrations")] + async fn test_create_sender_accounts_manager(pgpool: PgPool) { + let (_, (actor, join_handle)) = create_sender_accounts_manager(pgpool).await; + actor + .stop_and_wait(Some("Test".into()), None) + .await + .unwrap(); + join_handle.await.unwrap(); + } + + #[sqlx::test(migrations = "../migrations")] + async fn test_pending_sender_allocations(pgpool: PgPool) { + let config = get_config(); + let senders_to_signers = vec![(SENDER.1, vec![SIGNER.1])].into_iter().collect(); + let escrow_accounts = EscrowAccounts::new(HashMap::new(), senders_to_signers); + let state = State { + config, + domain_separator: TAP_EIP712_DOMAIN_SEPARATOR.clone(), + sender_ids: HashSet::new(), + new_receipts_watcher_handle: tokio::spawn(async {}), + _eligible_allocations_senders_pipe: Eventual::from_value(()).pipe_async(|_| async {}), + pgpool: pgpool.clone(), + indexer_allocations: Eventual::from_value(HashSet::new()), + escrow_accounts: Eventual::from_value(escrow_accounts), + escrow_subgraph: get_subgraph_client(), + sender_aggregator_endpoints: HashMap::new(), + prefix: None, + }; + + // add receipts to the database + for i in 1..=10 { + let receipt = create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i, i.into()); + store_receipt(&pgpool, receipt.signed_receipt()) + .await + .unwrap(); + } + + // add non-final ravs + let signed_rav = create_rav(*ALLOCATION_ID_1, SIGNER.0.clone(), 4, 10); + store_rav(&pgpool, signed_rav, SENDER.1).await.unwrap(); - #[test] - fn test_create_with_pending_sender_allocations() {} + let pending_allocation_id = state.get_pending_sender_allocation_id().await; - #[test] - fn test_update_sender_allocation() {} + // check if pending allocations are correct + assert_eq!(pending_allocation_id.len(), 1); + assert!(pending_allocation_id.get(&SENDER.1).is_some()); + assert_eq!(pending_allocation_id.get(&SENDER.1).unwrap().len(), 2); + } - #[test] - fn test_create_sender_account() {} + #[sqlx::test(migrations = "../migrations")] + async fn test_update_sender_allocation(pgpool: PgPool) { + let (prefix, (actor, join_handle)) = create_sender_accounts_manager(pgpool).await; + + actor + .cast(SenderAccountsManagerMessage::UpdateSenderAccounts( + vec![SENDER.1].into_iter().collect(), + )) + .unwrap(); + + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + + // verify if create sender account + let actor_ref = ActorRef::::where_is(format!( + "{}:{}", + prefix.clone(), + SENDER.1.to_string() + )); + assert!(actor_ref.is_some()); + + actor + .cast(SenderAccountsManagerMessage::UpdateSenderAccounts( + vec![].into_iter().collect(), + )) + .unwrap(); + + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + // verify if it gets removed + let actor_ref = ActorRef::::where_is(format!( + "{}:{}", + prefix, + SENDER.1.to_string() + )); + assert!(actor_ref.is_none()); + + // safely stop the manager + actor + .stop_and_wait(Some("Test".into()), None) + .await + .unwrap(); + join_handle.await.unwrap(); + } + + #[sqlx::test(migrations = "../migrations")] + async fn test_create_sender_account(pgpool: PgPool) { + let (prefix, (actor, join_handle)) = create_sender_accounts_manager(pgpool).await; + actor + .cast(SenderAccountsManagerMessage::CreateSenderAccount( + SENDER_2.1, + HashSet::new(), + )) + .unwrap(); + // we wait to check if the sender is created + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + + let actor_ref = ActorRef::::where_is(format!( + "{}:{}", + prefix, + SENDER_2.1.to_string() + )); + assert!(actor_ref.is_some()); + + actor + .stop_and_wait(Some("Test".into()), None) + .await + .unwrap(); + join_handle.await.unwrap(); + } } diff --git a/tap-agent/src/agent/sender_allocation.rs b/tap-agent/src/agent/sender_allocation.rs index 10982fc61..f9137bed4 100644 --- a/tap-agent/src/agent/sender_allocation.rs +++ b/tap-agent/src/agent/sender_allocation.rs @@ -709,7 +709,7 @@ mod tests { } #[sqlx::test(migrations = "../migrations")] - fn test_receive_new_receipt(pgpool: PgPool) { + async fn test_receive_new_receipt(pgpool: PgPool) { let (last_message_emitted, sender_account, _join_handle) = create_mock_sender_account().await; @@ -762,7 +762,7 @@ mod tests { } #[sqlx::test(migrations = "../migrations")] - fn test_trigger_rav_request(pgpool: PgPool) { + async fn test_trigger_rav_request(pgpool: PgPool) { // Start a TAP aggregator server. let (handle, aggregator_endpoint) = run_server( 0, @@ -1028,7 +1028,7 @@ mod tests { } #[sqlx::test(migrations = "../migrations")] - fn test_store_invalid_receipts(pgpool: PgPool) { + async fn test_store_invalid_receipts(pgpool: PgPool) { struct FailingCheck; #[async_trait::async_trait] @@ -1065,7 +1065,7 @@ mod tests { } #[sqlx::test(migrations = "../migrations")] - fn test_mark_rav_final(pgpool: PgPool) { + async fn test_mark_rav_final(pgpool: PgPool) { let signed_rav = create_rav(*ALLOCATION_ID_0, SIGNER.0.clone(), 4, 10); store_rav(&pgpool, signed_rav, SENDER.1).await.unwrap(); diff --git a/tap-agent/src/tap/test_utils.rs b/tap-agent/src/tap/test_utils.rs index 9c2a82066..0a1f06c09 100644 --- a/tap-agent/src/tap/test_utils.rs +++ b/tap-agent/src/tap/test_utils.rs @@ -22,7 +22,10 @@ use thegraph::types::Address; lazy_static! { pub static ref ALLOCATION_ID_0: Address = Address::from_str("0xabababababababababababababababababababab").unwrap(); + pub static ref ALLOCATION_ID_1: Address = + Address::from_str("0xbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc").unwrap(); pub static ref SENDER: (LocalWallet, Address) = wallet(0); + pub static ref SENDER_2: (LocalWallet, Address) = wallet(1); pub static ref SIGNER: (LocalWallet, Address) = wallet(2); pub static ref INDEXER: (LocalWallet, Address) = wallet(3); pub static ref TAP_EIP712_DOMAIN_SEPARATOR: Eip712Domain = eip712_domain! { diff --git a/tap-agent/tests/sender_manager_tests.rs b/tap-agent/tests/sender_manager_tests.rs deleted file mode 100644 index 46fbefc23..000000000 --- a/tap-agent/tests/sender_manager_tests.rs +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright 2023-, GraphOps and Semiotic Labs. -// SPDX-License-Identifier: Apache-2.0 - -use std::{collections::HashMap, str::FromStr, vec}; - -use alloy_primitives::Address; -use ethereum_types::U256; -use eventuals::Eventual; -use indexer_common::{ - allocations::Allocation, - escrow_accounts::EscrowAccounts, - prelude::{AllocationStatus, SubgraphDeployment}, - subgraph_client::{DeploymentDetails, SubgraphClient}, -}; -use indexer_tap_agent::{ - agent::{ - sender_account::SenderAccountMessage, - sender_accounts_manager::{SenderAccountsManager, SenderAccountsManagerArgs}, - sender_allocation::SenderAllocationMessage, - }, - config, -}; -use ractor::{Actor, ActorRef, ActorStatus}; -use serde_json::json; -use sqlx::PgPool; -use thegraph::types::DeploymentId; -use wiremock::{ - matchers::{body_string_contains, method}, - Mock, MockServer, ResponseTemplate, -}; - -mod test_utils; -use test_utils::{INDEXER, SENDER, SIGNER, TAP_EIP712_DOMAIN_SEPARATOR}; - -#[sqlx::test(migrations = "../migrations")] -async fn test_sender_account_creation_and_eol(pgpool: PgPool) { - let config = Box::leak(Box::new(config::Cli { - config: None, - ethereum: config::Ethereum { - indexer_address: INDEXER.1, - }, - tap: config::Tap { - rav_request_trigger_value: 100, - rav_request_timestamp_buffer_ms: 1, - ..Default::default() - }, - ..Default::default() - })); - - let (mut indexer_allocations_writer, indexer_allocations_eventual) = - Eventual::>::new(); - indexer_allocations_writer.write(HashMap::new()); - - let (mut escrow_accounts_writer, escrow_accounts_eventual) = Eventual::::new(); - escrow_accounts_writer.write(EscrowAccounts::default()); - - // Mock escrow subgraph. - let mock_server = MockServer::start().await; - mock_server - .register( - Mock::given(method("POST")) - .and(body_string_contains("transactions")) - .respond_with( - ResponseTemplate::new(200) - .set_body_json(json!({ "data": { "transactions": []}})), - ), - ) - .await; - let escrow_subgraph = Box::leak(Box::new(SubgraphClient::new( - reqwest::Client::new(), - None, - DeploymentDetails::for_query_url(&mock_server.uri()).unwrap(), - ))); - - let args = SenderAccountsManagerArgs { - config, - domain_separator: TAP_EIP712_DOMAIN_SEPARATOR.clone(), - pgpool: pgpool.clone(), - indexer_allocations: indexer_allocations_eventual, - escrow_accounts: escrow_accounts_eventual, - escrow_subgraph, - sender_aggregator_endpoints: HashMap::from([( - SENDER.1, - String::from("http://localhost:8000"), - )]), - }; - SenderAccountsManager::spawn(None, SenderAccountsManager, args) - .await - .unwrap(); - - let allocation_id = Address::from_str("0xdd975e30aafebb143e54d215db8a3e8fd916a701").unwrap(); - - // Add an allocation to the indexer_allocations Eventual. - indexer_allocations_writer.write(HashMap::from([( - allocation_id, - Allocation { - id: allocation_id, - indexer: INDEXER.1, - allocated_tokens: U256::from_str("601726452999999979510903").unwrap(), - created_at_block_hash: - "0x99d3fbdc0105f7ccc0cd5bb287b82657fe92db4ea8fb58242dafb90b1c6e2adf".to_string(), - created_at_epoch: 953, - closed_at_epoch: None, - subgraph_deployment: SubgraphDeployment { - id: DeploymentId::from_str( - "0xcda7fa0405d6fd10721ed13d18823d24b535060d8ff661f862b26c23334f13bf", - ) - .unwrap(), - denied_at: Some(0), - }, - status: AllocationStatus::Null, - closed_at_epoch_start_block_hash: None, - previous_epoch_start_block_hash: None, - poi: None, - query_fee_rebates: None, - query_fees_collected: None, - }, - )])); - - // Add an escrow account to the escrow_accounts Eventual. - escrow_accounts_writer.write(EscrowAccounts::new( - HashMap::from([(SENDER.1, 1000.into())]), - HashMap::from([(SENDER.1, vec![SIGNER.1])]), - )); - - // Wait for the SenderAccount to be created. - tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; - - // Check that the SenderAccount was created. - let sender_account = ActorRef::::where_is(SENDER.1.to_string()).unwrap(); - assert_eq!(sender_account.get_status(), ActorStatus::Running); - - let sender_allocation_id = format!("{}:{}", SENDER.1, allocation_id); - let sender_allocation = - ActorRef::::where_is(sender_allocation_id).unwrap(); - assert_eq!(sender_allocation.get_status(), ActorStatus::Running); - - // Remove the escrow account from the escrow_accounts Eventual. - escrow_accounts_writer.write(EscrowAccounts::default()); - - // Wait a bit - tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; - - // Check that the Sender's allocation moved from active to ineligible. - - assert_eq!(sender_account.get_status(), ActorStatus::Stopped); - assert_eq!(sender_allocation.get_status(), ActorStatus::Stopped); - - let sender_account = ActorRef::::where_is(SENDER.1.to_string()); - - assert!(sender_account.is_none()); -} From 9effaac87139223dc9976d6ae362705cac18641d Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Mon, 8 Apr 2024 23:49:31 -0300 Subject: [PATCH 21/41] style: fix clippy Signed-off-by: Gustavo Inacio --- tap-agent/src/agent/sender_account.rs | 30 +++++------ .../src/agent/sender_accounts_manager.rs | 51 ++++++++----------- 2 files changed, 36 insertions(+), 45 deletions(-) diff --git a/tap-agent/src/agent/sender_account.rs b/tap-agent/src/agent/sender_account.rs index a2b2eaead..0b65cd6d5 100644 --- a/tap-agent/src/agent/sender_account.rs +++ b/tap-agent/src/agent/sender_account.rs @@ -162,21 +162,6 @@ impl Actor for SenderAccount { }) } - async fn handle_supervisor_evt( - &self, - _myself: ActorRef, - message: SupervisionEvent, - _state: &mut Self::State, - ) -> std::result::Result<(), ActorProcessingErr> { - match message { - SupervisionEvent::ActorTerminated(_, _, _) | SupervisionEvent::ActorPanicked(_, _) => { - // what to do in case of termination or panic? - } - _ => {} - } - Ok(()) - } - async fn handle( &self, myself: ActorRef, @@ -243,6 +228,21 @@ impl Actor for SenderAccount { } Ok(()) } + + async fn handle_supervisor_evt( + &self, + _myself: ActorRef, + message: SupervisionEvent, + _state: &mut Self::State, + ) -> std::result::Result<(), ActorProcessingErr> { + match message { + SupervisionEvent::ActorTerminated(_, _, _) | SupervisionEvent::ActorPanicked(_, _) => { + // what to do in case of termination or panic? + } + _ => {} + } + Ok(()) + } } #[cfg(test)] diff --git a/tap-agent/src/agent/sender_accounts_manager.rs b/tap-agent/src/agent/sender_accounts_manager.rs index e3d1e2429..4a230990c 100644 --- a/tap-agent/src/agent/sender_accounts_manager.rs +++ b/tap-agent/src/agent/sender_accounts_manager.rs @@ -155,21 +155,6 @@ impl Actor for SenderAccountsManager { Ok(()) } - async fn handle_supervisor_evt( - &self, - _myself: ActorRef, - message: SupervisionEvent, - _state: &mut Self::State, - ) -> std::result::Result<(), ActorProcessingErr> { - match message { - SupervisionEvent::ActorTerminated(_, _, _) | SupervisionEvent::ActorPanicked(_, _) => { - // what to do in case of termination or panic - } - _ => {} - } - Ok(()) - } - async fn handle( &self, myself: ActorRef, @@ -210,6 +195,21 @@ impl Actor for SenderAccountsManager { } Ok(()) } + + async fn handle_supervisor_evt( + &self, + _myself: ActorRef, + message: SupervisionEvent, + _state: &mut Self::State, + ) -> std::result::Result<(), ActorProcessingErr> { + match message { + SupervisionEvent::ActorTerminated(_, _, _) | SupervisionEvent::ActorPanicked(_, _) => { + // what to do in case of termination or panic + } + _ => {} + } + Ok(()) + } } impl State { @@ -561,11 +561,8 @@ mod tests { tokio::time::sleep(std::time::Duration::from_millis(10)).await; // verify if create sender account - let actor_ref = ActorRef::::where_is(format!( - "{}:{}", - prefix.clone(), - SENDER.1.to_string() - )); + let actor_ref = + ActorRef::::where_is(format!("{}:{}", prefix.clone(), SENDER.1)); assert!(actor_ref.is_some()); actor @@ -576,11 +573,8 @@ mod tests { tokio::time::sleep(std::time::Duration::from_millis(10)).await; // verify if it gets removed - let actor_ref = ActorRef::::where_is(format!( - "{}:{}", - prefix, - SENDER.1.to_string() - )); + let actor_ref = + ActorRef::::where_is(format!("{}:{}", prefix, SENDER.1)); assert!(actor_ref.is_none()); // safely stop the manager @@ -603,11 +597,8 @@ mod tests { // we wait to check if the sender is created tokio::time::sleep(std::time::Duration::from_millis(10)).await; - let actor_ref = ActorRef::::where_is(format!( - "{}:{}", - prefix, - SENDER_2.1.to_string() - )); + let actor_ref = + ActorRef::::where_is(format!("{}:{}", prefix, SENDER_2.1)); assert!(actor_ref.is_some()); actor From bd630a3872a59f84e7b416adecc85f15f7962c15 Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Mon, 8 Apr 2024 23:49:49 -0300 Subject: [PATCH 22/41] chore: add licence to file Signed-off-by: Gustavo Inacio --- tap-agent/src/lib.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tap-agent/src/lib.rs b/tap-agent/src/lib.rs index 5782273ae..3e8258508 100644 --- a/tap-agent/src/lib.rs +++ b/tap-agent/src/lib.rs @@ -1,3 +1,6 @@ +// Copyright 2023-, GraphOps and Semiotic Labs. +// SPDX-License-Identifier: Apache-2.0 + use alloy_sol_types::{eip712_domain, Eip712Domain}; use lazy_static::lazy_static; From f1fdfd6f004a35e551e641fc77ded457332f56e2 Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Tue, 9 Apr 2024 17:28:37 -0300 Subject: [PATCH 23/41] chore: add comments --- tap-agent/src/agent/sender_account.rs | 106 +++++++++++++++--- .../src/agent/sender_accounts_manager.rs | 17 +-- 2 files changed, 98 insertions(+), 25 deletions(-) diff --git a/tap-agent/src/agent/sender_account.rs b/tap-agent/src/agent/sender_account.rs index 0b65cd6d5..b1a4279db 100644 --- a/tap-agent/src/agent/sender_account.rs +++ b/tap-agent/src/agent/sender_account.rs @@ -229,6 +229,8 @@ impl Actor for SenderAccount { Ok(()) } + // we define the supervisor event to overwrite the default behavior which + // is shutdown the supervisor on actor termination events async fn handle_supervisor_evt( &self, _myself: ActorRef, @@ -247,7 +249,18 @@ impl Actor for SenderAccount { #[cfg(test)] mod tests { - use super::SenderAccountMessage; + use super::{SenderAccount, SenderAccountArgs, SenderAccountMessage}; + use crate::config; + use crate::tap::test_utils::{INDEXER, SENDER, SIGNER, TAP_EIP712_DOMAIN_SEPARATOR}; + use eventuals::Eventual; + use indexer_common::escrow_accounts::EscrowAccounts; + use indexer_common::prelude::{DeploymentDetails, SubgraphClient}; + use ractor::concurrency::JoinHandle; + use ractor::ActorRef; + use sqlx::PgPool; + use std::collections::{HashMap, HashSet}; + use std::sync::atomic::AtomicU32; + use std::time::Duration; // we implement the PartialEq and Eq traits for SenderAccountMessage to be able to compare impl Eq for SenderAccountMessage {} @@ -265,21 +278,88 @@ mod tests { } } - #[test] - fn test_update_allocation_ids() {} + static PREFIX_ID: AtomicU32 = AtomicU32::new(0); + const DUMMY_URL: &str = "http://localhost:1234"; + const VALUE_PER_RECEIPT: u64 = 100; + const TRIGGER_VALUE: u64 = 500; + + async fn create_sender_with_allocations( + pgpool: PgPool, + sender_aggregator_endpoint: String, + escrow_subgraph_endpoint: &str, + ) -> ( + ActorRef, + tokio::task::JoinHandle<()>, + String, + ) { + let config = Box::leak(Box::new(config::Cli { + config: None, + ethereum: config::Ethereum { + indexer_address: INDEXER.1, + }, + tap: config::Tap { + rav_request_trigger_value: TRIGGER_VALUE, + rav_request_timestamp_buffer_ms: 1, + rav_request_timeout_secs: 5, + ..Default::default() + }, + ..Default::default() + })); + + let escrow_subgraph = Box::leak(Box::new(SubgraphClient::new( + reqwest::Client::new(), + None, + DeploymentDetails::for_query_url(escrow_subgraph_endpoint).unwrap(), + ))); + + let escrow_accounts_eventual = Eventual::from_value(EscrowAccounts::new( + HashMap::from([(SENDER.1, 1000.into())]), + HashMap::from([(SENDER.1, vec![SIGNER.1])]), + )); + + let prefix = format!( + "test-{}", + PREFIX_ID.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + ); + + let args = SenderAccountArgs { + config, + pgpool, + sender_id: SENDER.1, + escrow_accounts: escrow_accounts_eventual, + indexer_allocations: Eventual::from_value(HashSet::new()), + escrow_subgraph, + domain_separator: TAP_EIP712_DOMAIN_SEPARATOR.clone(), + sender_aggregator_endpoint, + allocation_ids: HashSet::new(), + prefix: Some(prefix.clone()), + }; + + let (sender, handle) = SenderAccount::spawn(Some(prefix.clone()), SenderAccount, args) + .await + .unwrap(); + (sender, handle, prefix) + } + + fn create_sender_account(pgpool: PgPool) -> (ActorRef, JoinHandle<()>) { + todo!() + } + + #[sqlx::test(migrations = "../migrations")] + async fn test_update_allocation_ids(pgpool: PgPool) {} - #[test] - fn test_update_receipt_fees_no_rav() {} + #[sqlx::test(migrations = "../migrations")] + async fn test_update_receipt_fees_no_rav(pgpool: PgPool) {} - #[test] - fn test_update_receipt_fees_trigger_rav() {} + #[sqlx::test(migrations = "../migrations")] + async fn test_update_receipt_fees_trigger_rav(pgpool: PgPool) {} - #[test] - fn test_remove_sender_account() {} + #[sqlx::test(migrations = "../migrations")] + async fn test_remove_sender_account(pgpool: PgPool) {} - #[test] - fn test_create_sender_allocation() {} + #[sqlx::test(migrations = "../migrations")] + async fn test_create_sender_allocation(pgpool: PgPool) {} - #[test] - fn test_create_sender_account() {} + #[sqlx::test(migrations = "../migrations")] + async fn test_create_sender_account(pgpool: PgPool) {} } diff --git a/tap-agent/src/agent/sender_accounts_manager.rs b/tap-agent/src/agent/sender_accounts_manager.rs index 4a230990c..9c0b79f1d 100644 --- a/tap-agent/src/agent/sender_accounts_manager.rs +++ b/tap-agent/src/agent/sender_accounts_manager.rs @@ -196,6 +196,8 @@ impl Actor for SenderAccountsManager { Ok(()) } + // we define the supervisor event to overwrite the default behavior which + // is shutdown the supervisor on actor termination events async fn handle_supervisor_evt( &self, _myself: ActorRef, @@ -502,10 +504,7 @@ mod tests { #[sqlx::test(migrations = "../migrations")] async fn test_create_sender_accounts_manager(pgpool: PgPool) { let (_, (actor, join_handle)) = create_sender_accounts_manager(pgpool).await; - actor - .stop_and_wait(Some("Test".into()), None) - .await - .unwrap(); + actor.stop_and_wait(None, None).await.unwrap(); join_handle.await.unwrap(); } @@ -578,10 +577,7 @@ mod tests { assert!(actor_ref.is_none()); // safely stop the manager - actor - .stop_and_wait(Some("Test".into()), None) - .await - .unwrap(); + actor.stop_and_wait(None, None).await.unwrap(); join_handle.await.unwrap(); } @@ -601,10 +597,7 @@ mod tests { ActorRef::::where_is(format!("{}:{}", prefix, SENDER_2.1)); assert!(actor_ref.is_some()); - actor - .stop_and_wait(Some("Test".into()), None) - .await - .unwrap(); + actor.stop_and_wait(None, None).await.unwrap(); join_handle.await.unwrap(); } } From d04c258abc0eb856f9890480cde777e505d49be8 Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Thu, 11 Apr 2024 13:39:48 -0300 Subject: [PATCH 24/41] test: add unit tests for sender_account Signed-off-by: Gustavo Inacio --- tap-agent/src/agent/sender_account.rs | 213 ++++++- .../src/agent/sender_accounts_manager.rs | 2 +- tap-agent/src/agent/sender_allocation.rs | 2 + tap-agent/tests/integration_test.rs | 143 +++++ tap-agent/tests/sender_account_tests.rs | 551 ------------------ tap-agent/tests/test_utils.rs | 140 ----- 6 files changed, 335 insertions(+), 716 deletions(-) create mode 100644 tap-agent/tests/integration_test.rs delete mode 100644 tap-agent/tests/sender_account_tests.rs delete mode 100644 tap-agent/tests/test_utils.rs diff --git a/tap-agent/src/agent/sender_account.rs b/tap-agent/src/agent/sender_account.rs index b1a4279db..f88f37fd7 100644 --- a/tap-agent/src/agent/sender_account.rs +++ b/tap-agent/src/agent/sender_account.rs @@ -27,6 +27,7 @@ pub enum SenderAccountMessage { UpdateAllocationIds(HashSet
), RemoveSenderAccount, UpdateReceiptFees(Address, UnaggregatedReceipts), + #[cfg(test)] GetAllocationTracker(ractor::RpcReplyPort), } @@ -219,7 +220,7 @@ impl Actor for SenderAccount { state.allocation_ids = allocation_ids; } - // #[cfg(test)] + #[cfg(test)] SenderAccountMessage::GetAllocationTracker(reply) => { if !reply.is_closed() { let _ = reply.send(state.allocation_id_tracker.clone()); @@ -250,16 +251,22 @@ impl Actor for SenderAccount { #[cfg(test)] mod tests { use super::{SenderAccount, SenderAccountArgs, SenderAccountMessage}; + use crate::agent::sender_allocation::SenderAllocationMessage; + use crate::agent::unaggregated_receipts::UnaggregatedReceipts; use crate::config; - use crate::tap::test_utils::{INDEXER, SENDER, SIGNER, TAP_EIP712_DOMAIN_SEPARATOR}; + use crate::tap::test_utils::{ + ALLOCATION_ID_0, INDEXER, SENDER, SIGNER, TAP_EIP712_DOMAIN_SEPARATOR, + }; + use alloy_primitives::Address; use eventuals::Eventual; use indexer_common::escrow_accounts::EscrowAccounts; use indexer_common::prelude::{DeploymentDetails, SubgraphClient}; use ractor::concurrency::JoinHandle; - use ractor::ActorRef; + use ractor::{Actor, ActorProcessingErr, ActorRef, ActorStatus}; use sqlx::PgPool; use std::collections::{HashMap, HashSet}; - use std::sync::atomic::AtomicU32; + use std::sync::atomic::{AtomicBool, AtomicU32}; + use std::sync::Arc; use std::time::Duration; // we implement the PartialEq and Eq traits for SenderAccountMessage to be able to compare @@ -280,13 +287,12 @@ mod tests { static PREFIX_ID: AtomicU32 = AtomicU32::new(0); const DUMMY_URL: &str = "http://localhost:1234"; - const VALUE_PER_RECEIPT: u64 = 100; - const TRIGGER_VALUE: u64 = 500; + const VALUE_PER_RECEIPT: u128 = 100; + const TRIGGER_VALUE: u128 = 500; - async fn create_sender_with_allocations( + async fn create_sender_account( pgpool: PgPool, - sender_aggregator_endpoint: String, - escrow_subgraph_endpoint: &str, + initial_allocation: HashSet
, ) -> ( ActorRef, tokio::task::JoinHandle<()>, @@ -298,7 +304,7 @@ mod tests { indexer_address: INDEXER.1, }, tap: config::Tap { - rav_request_trigger_value: TRIGGER_VALUE, + rav_request_trigger_value: TRIGGER_VALUE as u64, rav_request_timestamp_buffer_ms: 1, rav_request_timeout_secs: 5, ..Default::default() @@ -309,7 +315,7 @@ mod tests { let escrow_subgraph = Box::leak(Box::new(SubgraphClient::new( reqwest::Client::new(), None, - DeploymentDetails::for_query_url(escrow_subgraph_endpoint).unwrap(), + DeploymentDetails::for_query_url(DUMMY_URL).unwrap(), ))); let escrow_accounts_eventual = Eventual::from_value(EscrowAccounts::new( @@ -327,10 +333,10 @@ mod tests { pgpool, sender_id: SENDER.1, escrow_accounts: escrow_accounts_eventual, - indexer_allocations: Eventual::from_value(HashSet::new()), + indexer_allocations: Eventual::from_value(initial_allocation), escrow_subgraph, domain_separator: TAP_EIP712_DOMAIN_SEPARATOR.clone(), - sender_aggregator_endpoint, + sender_aggregator_endpoint: DUMMY_URL.to_string(), allocation_ids: HashSet::new(), prefix: Some(prefix.clone()), }; @@ -338,28 +344,187 @@ mod tests { let (sender, handle) = SenderAccount::spawn(Some(prefix.clone()), SenderAccount, args) .await .unwrap(); + tokio::time::sleep(Duration::from_millis(10)).await; (sender, handle, prefix) } - fn create_sender_account(pgpool: PgPool) -> (ActorRef, JoinHandle<()>) { - todo!() + #[sqlx::test(migrations = "../migrations")] + async fn test_update_allocation_ids(pgpool: PgPool) { + let (sender_account, handle, prefix) = create_sender_account(pgpool, HashSet::new()).await; + + // we expect it to create a sender allocation + sender_account + .cast(SenderAccountMessage::UpdateAllocationIds( + vec![*ALLOCATION_ID_0].into_iter().collect(), + )) + .unwrap(); + + tokio::time::sleep(Duration::from_millis(10)).await; + + // verify if create sender account + let sender_allocation_id = format!("{}:{}:{}", prefix.clone(), SENDER.1, *ALLOCATION_ID_0); + let actor_ref = ActorRef::::where_is(sender_allocation_id.clone()); + assert!(actor_ref.is_some()); + + sender_account + .cast(SenderAccountMessage::UpdateAllocationIds(HashSet::new())) + .unwrap(); + + tokio::time::sleep(Duration::from_millis(100)).await; + + let actor_ref = ActorRef::::where_is(sender_allocation_id.clone()); + assert!(actor_ref.is_none()); + + // safely stop the manager + sender_account.stop_and_wait(None, None).await.unwrap(); + + handle.await.unwrap(); } - #[sqlx::test(migrations = "../migrations")] - async fn test_update_allocation_ids(pgpool: PgPool) {} + struct MockSenderAllocation { + triggered_rav_request: Arc, + } - #[sqlx::test(migrations = "../migrations")] - async fn test_update_receipt_fees_no_rav(pgpool: PgPool) {} + #[async_trait::async_trait] + impl Actor for MockSenderAllocation { + type Msg = SenderAllocationMessage; + type State = (); + type Arguments = (); + + async fn pre_start( + &self, + _myself: ActorRef, + _allocation_ids: Self::Arguments, + ) -> Result { + Ok(()) + } - #[sqlx::test(migrations = "../migrations")] - async fn test_update_receipt_fees_trigger_rav(pgpool: PgPool) {} + async fn handle( + &self, + _myself: ActorRef, + message: Self::Msg, + _state: &mut Self::State, + ) -> Result<(), ActorProcessingErr> { + match message { + SenderAllocationMessage::TriggerRAVRequest(reply) => { + self.triggered_rav_request + .store(true, std::sync::atomic::Ordering::SeqCst); + reply.send(UnaggregatedReceipts::default())?; + } + _ => {} + } + Ok(()) + } + } + + async fn create_mock_sender_allocation( + prefix: String, + sender: Address, + allocation: Address, + ) -> ( + Arc, + ActorRef, + JoinHandle<()>, + ) { + let triggered_rav_request = Arc::new(AtomicBool::new(false)); + + let name = format!("{}:{}:{}", prefix, sender, allocation); + let (sender_account, join_handle) = MockSenderAllocation::spawn( + Some(name), + MockSenderAllocation { + triggered_rav_request: triggered_rav_request.clone(), + }, + (), + ) + .await + .unwrap(); + (triggered_rav_request, sender_account, join_handle) + } #[sqlx::test(migrations = "../migrations")] - async fn test_remove_sender_account(pgpool: PgPool) {} + async fn test_update_receipt_fees_no_rav(pgpool: PgPool) { + let (sender_account, handle, prefix) = create_sender_account(pgpool, HashSet::new()).await; + + let (triggered_rav_request, allocation, allocation_handle) = + create_mock_sender_allocation(prefix, SENDER.1, *ALLOCATION_ID_0).await; + + // create a fake sender allocation + sender_account + .cast(SenderAccountMessage::UpdateReceiptFees( + *ALLOCATION_ID_0, + UnaggregatedReceipts { + value: VALUE_PER_RECEIPT, + last_id: 10, + }, + )) + .unwrap(); + + tokio::time::sleep(Duration::from_millis(10)).await; + + assert!(!triggered_rav_request.load(std::sync::atomic::Ordering::SeqCst)); + + allocation.stop_and_wait(None, None).await.unwrap(); + allocation_handle.await.unwrap(); + + sender_account.stop_and_wait(None, None).await.unwrap(); + handle.await.unwrap(); + } #[sqlx::test(migrations = "../migrations")] - async fn test_create_sender_allocation(pgpool: PgPool) {} + async fn test_update_receipt_fees_trigger_rav(pgpool: PgPool) { + let (sender_account, handle, prefix) = create_sender_account(pgpool, HashSet::new()).await; + + let (triggered_rav_request, allocation, allocation_handle) = + create_mock_sender_allocation(prefix, SENDER.1, *ALLOCATION_ID_0).await; + + // create a fake sender allocation + sender_account + .cast(SenderAccountMessage::UpdateReceiptFees( + *ALLOCATION_ID_0, + UnaggregatedReceipts { + value: TRIGGER_VALUE, + last_id: 10, + }, + )) + .unwrap(); + + tokio::time::sleep(Duration::from_millis(10)).await; + + assert!(triggered_rav_request.load(std::sync::atomic::Ordering::SeqCst)); + + allocation.stop_and_wait(None, None).await.unwrap(); + allocation_handle.await.unwrap(); + + sender_account.stop_and_wait(None, None).await.unwrap(); + handle.await.unwrap(); + } #[sqlx::test(migrations = "../migrations")] - async fn test_create_sender_account(pgpool: PgPool) {} + async fn test_remove_sender_account(pgpool: PgPool) { + let (sender_account, handle, prefix) = + create_sender_account(pgpool, vec![*ALLOCATION_ID_0].into_iter().collect()).await; + + // check if allocation exists + let sender_allocation_id = format!("{}:{}:{}", prefix.clone(), SENDER.1, *ALLOCATION_ID_0); + let Some(sender_allocation) = + ActorRef::::where_is(sender_allocation_id.clone()) + else { + panic!("Sender allocation was not created"); + }; + + // stop + sender_account + .cast(SenderAccountMessage::RemoveSenderAccount) + .unwrap(); + tokio::time::sleep(Duration::from_millis(10)).await; + + // check if sender_account is stopped + + assert_eq!(sender_account.get_status(), ActorStatus::Stopped); + + // check if sender_allocation is also stopped + assert_eq!(sender_allocation.get_status(), ActorStatus::Stopped); + + handle.await.unwrap(); + } } diff --git a/tap-agent/src/agent/sender_accounts_manager.rs b/tap-agent/src/agent/sender_accounts_manager.rs index 9c0b79f1d..508fd4224 100644 --- a/tap-agent/src/agent/sender_accounts_manager.rs +++ b/tap-agent/src/agent/sender_accounts_manager.rs @@ -566,7 +566,7 @@ mod tests { actor .cast(SenderAccountsManagerMessage::UpdateSenderAccounts( - vec![].into_iter().collect(), + HashSet::new(), )) .unwrap(); diff --git a/tap-agent/src/agent/sender_allocation.rs b/tap-agent/src/agent/sender_allocation.rs index f9137bed4..718e992f2 100644 --- a/tap-agent/src/agent/sender_allocation.rs +++ b/tap-agent/src/agent/sender_allocation.rs @@ -69,6 +69,7 @@ pub enum SenderAllocationMessage { NewReceipt(NewReceiptNotification), TriggerRAVRequest(RpcReplyPort), CloseAllocation, + #[cfg(test)] GetUnaggregatedReceipts(RpcReplyPort), } @@ -181,6 +182,7 @@ impl Actor for SenderAllocation { // stop and trigger post_stop myself.stop(None); } + #[cfg(test)] SenderAllocationMessage::GetUnaggregatedReceipts(reply) => { if !reply.is_closed() { let _ = reply.send(unaggreated_fees.clone()); diff --git a/tap-agent/tests/integration_test.rs b/tap-agent/tests/integration_test.rs new file mode 100644 index 000000000..ed9d55282 --- /dev/null +++ b/tap-agent/tests/integration_test.rs @@ -0,0 +1,143 @@ +// Copyright 2023-, GraphOps and Semiotic Labs. +// SPDX-License-Identifier: Apache-2.0 +#[allow(dead_code)] +use alloy_primitives::hex::ToHex; +use alloy_primitives::Address; +use alloy_sol_types::{eip712_domain, Eip712Domain}; +use bigdecimal::{num_bigint::ToBigInt, ToPrimitive}; +use ethers::prelude::coins_bip39::English; +use ethers_signers::{LocalWallet, MnemonicBuilder, Signer}; +use eventuals::Eventual; +use indexer_common::escrow_accounts::EscrowAccounts; +use indexer_common::subgraph_client::{DeploymentDetails, SubgraphClient}; +use indexer_tap_agent::agent::sender_account::{ + SenderAccount, SenderAccountArgs, SenderAccountMessage, +}; +use indexer_tap_agent::agent::sender_accounts_manager::NewReceiptNotification; +use indexer_tap_agent::agent::sender_allocation::SenderAllocationMessage; +use indexer_tap_agent::agent::unaggregated_receipts::UnaggregatedReceipts; +use indexer_tap_agent::config; +use lazy_static::lazy_static; +use ractor::{call, cast, Actor, ActorRef}; +use serde_json::json; +use sqlx::PgPool; +use std::collections::HashSet; +use std::str::FromStr; +use std::time::Duration; +use std::{collections::HashMap, sync::atomic::AtomicU32}; +use tap_aggregator::server::run_server; +use tap_core::{rav::ReceiptAggregateVoucher, signed_message::EIP712SignedMessage}; +use tokio::task::JoinHandle; +use wiremock::{ + matchers::{body_string_contains, method}, + Mock, MockServer, ResponseTemplate, +}; + +const DUMMY_URL: &str = "http://localhost:1234"; +const VALUE_PER_RECEIPT: u64 = 100; +const TRIGGER_VALUE: u64 = 500; +static PREFIX_ID: AtomicU32 = AtomicU32::new(0); + +lazy_static! { + pub static ref ALLOCATION_ID_0: Address = + Address::from_str("0xabababababababababababababababababababab").unwrap(); + pub static ref ALLOCATION_ID_1: Address = + Address::from_str("0xbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc").unwrap(); + pub static ref SENDER: (LocalWallet, Address) = wallet(0); + pub static ref SENDER_2: (LocalWallet, Address) = wallet(1); + pub static ref SIGNER: (LocalWallet, Address) = wallet(2); + pub static ref INDEXER: (LocalWallet, Address) = wallet(3); + pub static ref TAP_EIP712_DOMAIN_SEPARATOR: Eip712Domain = eip712_domain! { + name: "TAP", + version: "1", + chain_id: 1, + verifying_contract: Address:: from([0x11u8; 20]), + }; +} + +pub fn wallet(index: u32) -> (LocalWallet, Address) { + let wallet: LocalWallet = MnemonicBuilder::::default() + .phrase("abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon about") + .index(index) + .unwrap() + .build() + .unwrap(); + let address = wallet.address(); + (wallet, Address::from_slice(address.as_bytes())) +} + +async fn create_aggregator() { + // Start a TAP aggregator server. + let (handle, aggregator_endpoint) = run_server( + 0, + SIGNER.0.clone(), + vec![SIGNER.1].into_iter().collect(), + TAP_EIP712_DOMAIN_SEPARATOR.clone(), + 100 * 1024, + 100 * 1024, + 1, + ) + .await + .unwrap(); +} + +async fn create_mock_server() { + // Start a mock graphql server using wiremock + let mock_server = MockServer::start().await; + + // Mock result for TAP redeem txs for (allocation, sender) pair. + mock_server + .register( + Mock::given(method("POST")) + .and(body_string_contains("transactions")) + .respond_with( + ResponseTemplate::new(200) + .set_body_json(json!({ "data": { "transactions": []}})), + ), + ) + .await; +} + +async fn create_sender_accounts_manager() {} + +#[sqlx::test(migrations = "../migrations")] +async fn test_full_happy_path(pgpool: PgPool) { + + // create mock server + + // create mock server for getting escrow subgraph + + // create sender_allocation_manager + + // add sender account + + // add sender allocation 1 + // add sender allocation 2 + // add sender allocation 3 + + // store receipt + + // check if receipt was detected by notification handler + + // check if it was added on the sender_allocation + + // check if it was added on the sender_account + + // add more receipts on both until a trigger value is reached + + // check if it was triggered + + // check if there's a rav on the database + + // close the sender allocation 1 + + // check if the rav 1 is now final + + // close the sender account + + // check if the rav 2 is now final + + // kill the manager (simulating a shutdown of the system) + + // check if the rav 3 is still non-final +} diff --git a/tap-agent/tests/sender_account_tests.rs b/tap-agent/tests/sender_account_tests.rs deleted file mode 100644 index d7a1aa05c..000000000 --- a/tap-agent/tests/sender_account_tests.rs +++ /dev/null @@ -1,551 +0,0 @@ -// Copyright 2023-, GraphOps and Semiotic Labs. -// SPDX-License-Identifier: Apache-2.0 - -use alloy_primitives::hex::ToHex; -use alloy_primitives::Address; -use bigdecimal::{num_bigint::ToBigInt, ToPrimitive}; -use eventuals::Eventual; -use indexer_common::escrow_accounts::EscrowAccounts; -use indexer_common::subgraph_client::{DeploymentDetails, SubgraphClient}; -use indexer_tap_agent::agent::sender_account::{ - SenderAccount, SenderAccountArgs, SenderAccountMessage, -}; -use indexer_tap_agent::agent::sender_accounts_manager::NewReceiptNotification; -use indexer_tap_agent::agent::sender_allocation::SenderAllocationMessage; -use indexer_tap_agent::agent::unaggregated_receipts::UnaggregatedReceipts; -use indexer_tap_agent::config; -use ractor::{call, cast, Actor, ActorRef}; -use serde_json::json; -use sqlx::PgPool; -use std::collections::HashSet; -use std::str::FromStr; -use std::time::Duration; -use std::{collections::HashMap, sync::atomic::AtomicU32}; -use tap_aggregator::server::run_server; -use tap_core::{rav::ReceiptAggregateVoucher, signed_message::EIP712SignedMessage}; -use tokio::task::JoinHandle; -use wiremock::{ - matchers::{body_string_contains, method}, - Mock, MockServer, ResponseTemplate, -}; - -use test_utils::{ - create_received_receipt, store_receipt, ALLOCATION_ID_0, ALLOCATION_ID_1, ALLOCATION_ID_2, - INDEXER, SENDER, SIGNER, TAP_EIP712_DOMAIN_SEPARATOR, -}; - -mod test_utils; - -const DUMMY_URL: &str = "http://localhost:1234"; -const VALUE_PER_RECEIPT: u64 = 100; -const TRIGGER_VALUE: u64 = 500; -static PREFIX_ID: AtomicU32 = AtomicU32::new(0); - -// To help with testing from other modules. - -async fn create_sender_with_allocations( - pgpool: PgPool, - sender_aggregator_endpoint: String, - escrow_subgraph_endpoint: &str, -) -> (ActorRef, JoinHandle<()>, String) { - let config = Box::leak(Box::new(config::Cli { - config: None, - ethereum: config::Ethereum { - indexer_address: INDEXER.1, - }, - tap: config::Tap { - rav_request_trigger_value: TRIGGER_VALUE, - rav_request_timestamp_buffer_ms: 1, - rav_request_timeout_secs: 5, - ..Default::default() - }, - ..Default::default() - })); - - let escrow_subgraph = Box::leak(Box::new(SubgraphClient::new( - reqwest::Client::new(), - None, - DeploymentDetails::for_query_url(escrow_subgraph_endpoint).unwrap(), - ))); - - let escrow_accounts_eventual = Eventual::from_value(EscrowAccounts::new( - HashMap::from([(SENDER.1, 1000.into())]), - HashMap::from([(SENDER.1, vec![SIGNER.1])]), - )); - - let indexer_allocations = Eventual::from_value(HashSet::from([ - *ALLOCATION_ID_0, - *ALLOCATION_ID_1, - *ALLOCATION_ID_2, - ])); - - let prefix = format!( - "test-{}", - PREFIX_ID.fetch_add(1, std::sync::atomic::Ordering::SeqCst) - ); - - let args = SenderAccountArgs { - config, - pgpool, - sender_id: SENDER.1, - escrow_accounts: escrow_accounts_eventual, - indexer_allocations, - escrow_subgraph, - domain_separator: TAP_EIP712_DOMAIN_SEPARATOR.clone(), - sender_aggregator_endpoint, - allocation_ids: HashSet::new(), - prefix: Some(prefix.clone()), - }; - - let (sender, handle) = SenderAccount::spawn(Some(prefix.clone()), SenderAccount, args) - .await - .unwrap(); - - // await for the allocations to be created - ractor::concurrency::sleep(Duration::from_millis(100)).await; - - (sender, handle, prefix) -} - -/// Test that the sender_account correctly ignores new receipt notifications with -/// an ID lower than the last receipt ID processed (be it from the DB or from a prior receipt -/// notification). -#[sqlx::test(migrations = "../migrations")] -async fn test_handle_new_receipt_notification(pgpool: PgPool) { - // Add receipts to the database. Before creating the sender and allocation so that it loads - // the receipts from the DB. - let mut expected_unaggregated_fees = 0u128; - for i in 10..20 { - let receipt = create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i, i.into()).await; - store_receipt(&pgpool, receipt.signed_receipt()) - .await - .unwrap(); - expected_unaggregated_fees += u128::from(i); - } - - let (sender, handle, prefix) = - create_sender_with_allocations(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL).await; - - let allocation = ActorRef::::where_is(format!( - "{prefix}:{sender}:{allocation_id}", - sender = SENDER.1, - allocation_id = *ALLOCATION_ID_0 - )) - .unwrap(); - - // Check that the sender's unaggregated fees are correct. - let allocation_tracker = call!(sender, SenderAccountMessage::GetAllocationTracker).unwrap(); - assert_eq!( - allocation_tracker.get_total_fee(), - expected_unaggregated_fees - ); - - let allocation_unaggregated_fees = - call!(allocation, SenderAllocationMessage::GetUnaggregatedReceipts).unwrap(); - - // Check that the allocation's unaggregated fees are correct. - assert_eq!( - allocation_unaggregated_fees.value, - expected_unaggregated_fees - ); - - // Send a new receipt notification that has a lower ID than the last loaded from the DB. - // The last ID in the DB should be 10, since we added 10 receipts to the empty receipts - // table - let new_receipt_notification = NewReceiptNotification { - allocation_id: *ALLOCATION_ID_0, - signer_address: SIGNER.1, - id: 10, - timestamp_ns: 19, - value: 19, - }; - allocation - .cast(SenderAllocationMessage::NewReceipt( - new_receipt_notification, - )) - .unwrap(); - ractor::concurrency::sleep(Duration::from_millis(10)).await; - - let allocation_unaggregated_fees = - call!(allocation, SenderAllocationMessage::GetUnaggregatedReceipts).unwrap(); - - // Check that the allocation's unaggregated fees have *not* increased. - assert_eq!( - allocation_unaggregated_fees.value, - expected_unaggregated_fees - ); - - // Check that the unaggregated fees have *not* increased. - let allocation_tracker = call!(sender, SenderAccountMessage::GetAllocationTracker).unwrap(); - assert_eq!( - allocation_tracker.get_total_fee(), - expected_unaggregated_fees - ); - - // Send a new receipt notification. - let new_receipt_notification = NewReceiptNotification { - allocation_id: *ALLOCATION_ID_0, - signer_address: SIGNER.1, - id: 30, - timestamp_ns: 20, - value: 20, - }; - allocation - .cast(SenderAllocationMessage::NewReceipt( - new_receipt_notification, - )) - .unwrap(); - ractor::concurrency::sleep(Duration::from_millis(10)).await; - - expected_unaggregated_fees += 20; - - let allocation_unaggregated_fees = - call!(allocation, SenderAllocationMessage::GetUnaggregatedReceipts).unwrap(); - - // Check that the allocation's unaggregated fees are correct. - assert_eq!( - allocation_unaggregated_fees.value, - expected_unaggregated_fees - ); - - // Check that the sender's unaggregated fees are correct. - let allocation_tracker = call!(sender, SenderAccountMessage::GetAllocationTracker).unwrap(); - assert_eq!( - allocation_tracker.get_total_fee(), - expected_unaggregated_fees - ); - - // Send a new receipt notification that has a lower ID than the previous one. - let new_receipt_notification = NewReceiptNotification { - allocation_id: *ALLOCATION_ID_0, - signer_address: SIGNER.1, - id: 25, - timestamp_ns: 19, - value: 19, - }; - allocation - .cast(SenderAllocationMessage::NewReceipt( - new_receipt_notification, - )) - .unwrap(); - - ractor::concurrency::sleep(Duration::from_millis(10)).await; - - let allocation_unaggregated_fees = - call!(allocation, SenderAllocationMessage::GetUnaggregatedReceipts).unwrap(); - - // Check that the allocation's unaggregated fees have *not* increased. - assert_eq!( - allocation_unaggregated_fees.value, - expected_unaggregated_fees - ); - - // Check that the unaggregated fees have *not* increased. - let allocation_tracker = call!(sender, SenderAccountMessage::GetAllocationTracker).unwrap(); - assert_eq!( - allocation_tracker.get_total_fee(), - expected_unaggregated_fees - ); - - sender.stop(None); - handle.await.unwrap(); -} - -#[sqlx::test(migrations = "../migrations")] -async fn test_rav_requester_auto(pgpool: PgPool) { - // Start a TAP aggregator server. - let (handle, aggregator_endpoint) = run_server( - 0, - SIGNER.0.clone(), - vec![SIGNER.1].into_iter().collect(), - TAP_EIP712_DOMAIN_SEPARATOR.clone(), - 100 * 1024, - 100 * 1024, - 1, - ) - .await - .unwrap(); - - // Start a mock graphql server using wiremock - let mock_server = MockServer::start().await; - - // Mock result for TAP redeem txs for (allocation, sender) pair. - mock_server - .register( - Mock::given(method("POST")) - .and(body_string_contains("transactions")) - .respond_with( - ResponseTemplate::new(200) - .set_body_json(json!({ "data": { "transactions": []}})), - ), - ) - .await; - - // Create a sender_account. - let (sender_account, sender_handle, prefix) = create_sender_with_allocations( - pgpool.clone(), - "http://".to_owned() + &aggregator_endpoint.to_string(), - &mock_server.uri(), - ) - .await; - - let allocation = ActorRef::::where_is(format!( - "{prefix}:{sender}:{allocation_id}", - sender = SENDER.1, - allocation_id = *ALLOCATION_ID_0 - )) - .unwrap(); - - // Add receipts to the database and call the `handle_new_receipt_notification` method - // correspondingly. - let mut total_value = 0; - let mut trigger_value = 0; - for i in 1..=10 { - // These values should be enough to trigger a RAV request at i == 7 since we set the - // `rav_request_trigger_value` to 100. - let value = (i + VALUE_PER_RECEIPT) as u128; - - let receipt = create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i + 1, value).await; - store_receipt(&pgpool, receipt.signed_receipt()) - .await - .unwrap(); - let new_receipt_notification = NewReceiptNotification { - allocation_id: *ALLOCATION_ID_0, - signer_address: SIGNER.1, - id: i, - timestamp_ns: i + 1, - value, - }; - allocation - .cast(SenderAllocationMessage::NewReceipt( - new_receipt_notification, - )) - .unwrap(); - - ractor::concurrency::sleep(Duration::from_millis(100)).await; - - total_value += value; - if total_value >= TRIGGER_VALUE as u128 && trigger_value == 0 { - trigger_value = total_value; - } - } - - ractor::concurrency::sleep(Duration::from_millis(10)).await; - - // Wait for the RAV requester to finish. - for _ in 0..100 { - let allocation_tracker = - call!(sender_account, SenderAccountMessage::GetAllocationTracker).unwrap(); - if allocation_tracker.get_total_fee() < trigger_value { - break; - } - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - } - - // Get the latest RAV from the database. - let latest_rav = sqlx::query!( - r#" - SELECT signature, allocation_id, timestamp_ns, value_aggregate - FROM scalar_tap_ravs - WHERE allocation_id = $1 AND sender_address = $2 - "#, - ALLOCATION_ID_0.encode_hex::(), - SENDER.1.encode_hex::() - ) - .fetch_optional(&pgpool) - .await - .unwrap() - .unwrap(); - - let latest_rav = EIP712SignedMessage { - message: ReceiptAggregateVoucher { - allocationId: Address::from_str(&latest_rav.allocation_id).unwrap(), - timestampNs: latest_rav.timestamp_ns.to_u64().unwrap(), - // Beware, BigDecimal::to_u128() actually uses to_u64() under the hood... - // So we're converting to BigInt to get a proper implementation of to_u128(). - valueAggregate: latest_rav - .value_aggregate - .to_bigint() - .map(|v| v.to_u128()) - .unwrap() - .unwrap(), - }, - signature: latest_rav.signature.as_slice().try_into().unwrap(), - }; - - // Check that the latest RAV value is correct. - assert!(latest_rav.message.valueAggregate >= trigger_value); - - // Check that the allocation's unaggregated fees value is reduced. - - let allocation_unaggregated_fees = - call!(allocation, SenderAllocationMessage::GetUnaggregatedReceipts).unwrap(); - - // Check that the allocation's unaggregated fees have *not* increased. - assert!(allocation_unaggregated_fees.value <= trigger_value); - - // Check that the sender's unaggregated fees value is reduced. - let allocation_tracker = - call!(sender_account, SenderAccountMessage::GetAllocationTracker).unwrap(); - assert!(allocation_tracker.get_total_fee() <= trigger_value); - - // Reset the total value and trigger value. - total_value = allocation_tracker.get_total_fee(); - trigger_value = 0; - - // Add more receipts - for i in 10..20 { - let value = (i + VALUE_PER_RECEIPT) as u128; - - let receipt = - create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i + 1, i.into()).await; - store_receipt(&pgpool, receipt.signed_receipt()) - .await - .unwrap(); - - let new_receipt_notification = NewReceiptNotification { - allocation_id: *ALLOCATION_ID_0, - signer_address: SIGNER.1, - id: i, - timestamp_ns: i + 1, - value, - }; - allocation - .cast(SenderAllocationMessage::NewReceipt( - new_receipt_notification, - )) - .unwrap(); - - ractor::concurrency::sleep(Duration::from_millis(10)).await; - - total_value += value; - if total_value >= TRIGGER_VALUE as u128 && trigger_value == 0 { - trigger_value = total_value; - } - } - - // Wait for the RAV requester to finish. - for _ in 0..100 { - let allocation_tracker = - call!(sender_account, SenderAccountMessage::GetAllocationTracker).unwrap(); - if allocation_tracker.get_total_fee() < trigger_value { - break; - } - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - } - - // Get the latest RAV from the database. - let latest_rav = sqlx::query!( - r#" - SELECT signature, allocation_id, timestamp_ns, value_aggregate - FROM scalar_tap_ravs - WHERE allocation_id = $1 AND sender_address = $2 - "#, - ALLOCATION_ID_0.encode_hex::(), - SENDER.1.encode_hex::() - ) - .fetch_optional(&pgpool) - .await - .unwrap() - .unwrap(); - - let latest_rav = EIP712SignedMessage { - message: ReceiptAggregateVoucher { - allocationId: Address::from_str(&latest_rav.allocation_id).unwrap(), - timestampNs: latest_rav.timestamp_ns.to_u64().unwrap(), - // Beware, BigDecimal::to_u128() actually uses to_u64() under the hood... - // So we're converting to BigInt to get a proper implementation of to_u128(). - valueAggregate: latest_rav - .value_aggregate - .to_bigint() - .map(|v| v.to_u128()) - .unwrap() - .unwrap(), - }, - signature: latest_rav.signature.as_slice().try_into().unwrap(), - }; - - // Check that the latest RAV value is correct. - - assert!(latest_rav.message.valueAggregate >= trigger_value); - - // Check that the allocation's unaggregated fees value is reduced. - - let allocation_unaggregated_fees = - call!(allocation, SenderAllocationMessage::GetUnaggregatedReceipts).unwrap(); - - // Check that the allocation's unaggregated fees have *not* increased. - assert!(allocation_unaggregated_fees.value <= trigger_value); - - // Check that the unaggregated fees value is reduced. - let allocation_tracker = - call!(sender_account, SenderAccountMessage::GetAllocationTracker).unwrap(); - assert!(allocation_tracker.get_total_fee() <= trigger_value); - - // Stop the TAP aggregator server. - handle.stop().unwrap(); - handle.stopped().await; - - sender_account.stop(None); - sender_handle.await.unwrap(); -} - -#[sqlx::test(migrations = "../migrations")] -async fn test_sender_unaggregated_fees(pgpool: PgPool) { - // Create a sender_account. - let (sender_account, handle, _prefix) = - create_sender_with_allocations(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL).await; - - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - - // Closure that adds a number of receipts to an allocation. - - let update_receipt_fees = - |sender_account: &ActorRef, allocation_id: Address, value: u128| { - cast!( - sender_account, - SenderAccountMessage::UpdateReceiptFees( - allocation_id, - UnaggregatedReceipts { - value, - ..Default::default() - }, - ) - ) - .unwrap(); - - value - }; - - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - - // Add receipts to the database for allocation_0 - let total_value_0 = update_receipt_fees(&sender_account, *ALLOCATION_ID_0, 90); - tokio::time::sleep(std::time::Duration::from_millis(10)).await; - - // Add receipts to the database for allocation_1 - let total_value_1 = update_receipt_fees(&sender_account, *ALLOCATION_ID_1, 100); - tokio::time::sleep(std::time::Duration::from_millis(10)).await; - - // Add receipts to the database for allocation_2 - let total_value_2 = update_receipt_fees(&sender_account, *ALLOCATION_ID_2, 80); - tokio::time::sleep(std::time::Duration::from_millis(10)).await; - - let allocation_tracker = - call!(sender_account, SenderAccountMessage::GetAllocationTracker).unwrap(); - - // Get the heaviest allocation. - let heaviest_allocation = allocation_tracker.get_heaviest_allocation_id().unwrap(); - - // Check that the heaviest allocation is correct. - assert_eq!(heaviest_allocation, *ALLOCATION_ID_1); - - // Check that the sender's unaggregated fees value is correct. - assert_eq!( - allocation_tracker.get_total_fee(), - total_value_0 + total_value_1 + total_value_2 - ); - - sender_account.stop(None); - handle.await.unwrap(); -} diff --git a/tap-agent/tests/test_utils.rs b/tap-agent/tests/test_utils.rs deleted file mode 100644 index 9ddd7d11f..000000000 --- a/tap-agent/tests/test_utils.rs +++ /dev/null @@ -1,140 +0,0 @@ -// Copyright 2023-, GraphOps and Semiotic Labs. -// SPDX-License-Identifier: Apache-2.0 - -use std::str::FromStr; - -use alloy_primitives::hex::ToHex; -use alloy_sol_types::{eip712_domain, Eip712Domain}; -use anyhow::Result; -use bigdecimal::num_bigint::BigInt; -use ethers_signers::{coins_bip39::English, LocalWallet, MnemonicBuilder, Signer}; -use lazy_static::lazy_static; -use sqlx::{types::BigDecimal, PgPool}; -use tap_core::{ - rav::{ReceiptAggregateVoucher, SignedRAV}, - receipt::{Checking, Receipt, ReceiptWithState, SignedReceipt}, - signed_message::EIP712SignedMessage, -}; -use thegraph::types::Address; - -lazy_static! { - pub static ref ALLOCATION_ID_0: Address = - Address::from_str("0xabababababababababababababababababababab").unwrap(); - pub static ref ALLOCATION_ID_1: Address = - Address::from_str("0xbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc").unwrap(); - pub static ref ALLOCATION_ID_2: Address = - Address::from_str("0xcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd").unwrap(); - pub static ref ALLOCATION_ID_IRRELEVANT: Address = - Address::from_str("0xbcdebcdebcdebcdebcdebcdebcdebcdebcdebcde").unwrap(); - pub static ref SENDER: (LocalWallet, Address) = wallet(0); - pub static ref SENDER_IRRELEVANT: (LocalWallet, Address) = wallet(1); - pub static ref SIGNER: (LocalWallet, Address) = wallet(2); - pub static ref INDEXER: (LocalWallet, Address) = wallet(3); - pub static ref TAP_EIP712_DOMAIN_SEPARATOR: Eip712Domain = eip712_domain! { - name: "TAP", - version: "1", - chain_id: 1, - verifying_contract: Address:: from([0x11u8; 20]), - }; -} - -/// Fixture to generate a wallet and address -fn wallet(index: u32) -> (LocalWallet, Address) { - let wallet: LocalWallet = MnemonicBuilder::::default() - .phrase("abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon about") - .index(index) - .unwrap() - .build() - .unwrap(); - let address = wallet.address(); - (wallet, Address::from_slice(address.as_bytes())) -} - -/// Fixture to generate a signed receipt using the wallet from `keys()` and the -/// given `query_id` and `value` -pub async fn create_received_receipt( - allocation_id: &Address, - signer_wallet: &LocalWallet, - nonce: u64, - timestamp_ns: u64, - value: u128, -) -> ReceiptWithState { - let receipt = EIP712SignedMessage::new( - &TAP_EIP712_DOMAIN_SEPARATOR, - Receipt { - allocation_id: *allocation_id, - nonce, - timestamp_ns, - value, - }, - signer_wallet, - ) - .unwrap(); - ReceiptWithState::new(receipt) -} - -/// Fixture to generate a RAV using the wallet from `keys()` -pub async fn create_rav( - allocation_id: Address, - signer_wallet: LocalWallet, - timestamp_ns: u64, - value_aggregate: u128, -) -> SignedRAV { - EIP712SignedMessage::new( - &TAP_EIP712_DOMAIN_SEPARATOR, - ReceiptAggregateVoucher { - allocationId: allocation_id, - timestampNs: timestamp_ns, - valueAggregate: value_aggregate, - }, - &signer_wallet, - ) - .unwrap() -} - -pub async fn store_receipt(pgpool: &PgPool, signed_receipt: &SignedReceipt) -> Result { - let encoded_signature = signed_receipt.signature.to_vec(); - - let record = sqlx::query!( - r#" - INSERT INTO scalar_tap_receipts (signer_address, signature, allocation_id, timestamp_ns, nonce, value) - VALUES ($1, $2, $3, $4, $5, $6) - RETURNING id - "#, - signed_receipt - .recover_signer(&TAP_EIP712_DOMAIN_SEPARATOR) - .unwrap() - .encode_hex::(), - encoded_signature, - signed_receipt.message.allocation_id.encode_hex::(), - BigDecimal::from(signed_receipt.message.timestamp_ns), - BigDecimal::from(signed_receipt.message.nonce), - BigDecimal::from(BigInt::from(signed_receipt.message.value)), - ) - .fetch_one(pgpool) - .await?; - - // id is BIGSERIAL, so it should be safe to cast to u64. - let id: u64 = record.id.try_into()?; - Ok(id) -} - -pub async fn store_rav(pgpool: &PgPool, signed_rav: SignedRAV, sender: Address) -> Result<()> { - let signature_bytes = signed_rav.signature.to_vec(); - - let _fut = sqlx::query!( - r#" - INSERT INTO scalar_tap_ravs (sender_address, signature, allocation_id, timestamp_ns, value_aggregate) - VALUES ($1, $2, $3, $4, $5) - "#, - sender.encode_hex::(), - signature_bytes, - signed_rav.message.allocationId.encode_hex::(), - BigDecimal::from(signed_rav.message.timestampNs), - BigDecimal::from(BigInt::from(signed_rav.message.valueAggregate)), - ) - .execute(pgpool) - .await?; - - Ok(()) -} From 3ea3cda2ee1416584d81bfbd634308ca80639274 Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Thu, 11 Apr 2024 13:54:18 -0300 Subject: [PATCH 25/41] test: remove integration test initial file Signed-off-by: Gustavo Inacio --- tap-agent/tests/integration_test.rs | 143 ---------------------------- 1 file changed, 143 deletions(-) delete mode 100644 tap-agent/tests/integration_test.rs diff --git a/tap-agent/tests/integration_test.rs b/tap-agent/tests/integration_test.rs deleted file mode 100644 index ed9d55282..000000000 --- a/tap-agent/tests/integration_test.rs +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright 2023-, GraphOps and Semiotic Labs. -// SPDX-License-Identifier: Apache-2.0 -#[allow(dead_code)] -use alloy_primitives::hex::ToHex; -use alloy_primitives::Address; -use alloy_sol_types::{eip712_domain, Eip712Domain}; -use bigdecimal::{num_bigint::ToBigInt, ToPrimitive}; -use ethers::prelude::coins_bip39::English; -use ethers_signers::{LocalWallet, MnemonicBuilder, Signer}; -use eventuals::Eventual; -use indexer_common::escrow_accounts::EscrowAccounts; -use indexer_common::subgraph_client::{DeploymentDetails, SubgraphClient}; -use indexer_tap_agent::agent::sender_account::{ - SenderAccount, SenderAccountArgs, SenderAccountMessage, -}; -use indexer_tap_agent::agent::sender_accounts_manager::NewReceiptNotification; -use indexer_tap_agent::agent::sender_allocation::SenderAllocationMessage; -use indexer_tap_agent::agent::unaggregated_receipts::UnaggregatedReceipts; -use indexer_tap_agent::config; -use lazy_static::lazy_static; -use ractor::{call, cast, Actor, ActorRef}; -use serde_json::json; -use sqlx::PgPool; -use std::collections::HashSet; -use std::str::FromStr; -use std::time::Duration; -use std::{collections::HashMap, sync::atomic::AtomicU32}; -use tap_aggregator::server::run_server; -use tap_core::{rav::ReceiptAggregateVoucher, signed_message::EIP712SignedMessage}; -use tokio::task::JoinHandle; -use wiremock::{ - matchers::{body_string_contains, method}, - Mock, MockServer, ResponseTemplate, -}; - -const DUMMY_URL: &str = "http://localhost:1234"; -const VALUE_PER_RECEIPT: u64 = 100; -const TRIGGER_VALUE: u64 = 500; -static PREFIX_ID: AtomicU32 = AtomicU32::new(0); - -lazy_static! { - pub static ref ALLOCATION_ID_0: Address = - Address::from_str("0xabababababababababababababababababababab").unwrap(); - pub static ref ALLOCATION_ID_1: Address = - Address::from_str("0xbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc").unwrap(); - pub static ref SENDER: (LocalWallet, Address) = wallet(0); - pub static ref SENDER_2: (LocalWallet, Address) = wallet(1); - pub static ref SIGNER: (LocalWallet, Address) = wallet(2); - pub static ref INDEXER: (LocalWallet, Address) = wallet(3); - pub static ref TAP_EIP712_DOMAIN_SEPARATOR: Eip712Domain = eip712_domain! { - name: "TAP", - version: "1", - chain_id: 1, - verifying_contract: Address:: from([0x11u8; 20]), - }; -} - -pub fn wallet(index: u32) -> (LocalWallet, Address) { - let wallet: LocalWallet = MnemonicBuilder::::default() - .phrase("abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon about") - .index(index) - .unwrap() - .build() - .unwrap(); - let address = wallet.address(); - (wallet, Address::from_slice(address.as_bytes())) -} - -async fn create_aggregator() { - // Start a TAP aggregator server. - let (handle, aggregator_endpoint) = run_server( - 0, - SIGNER.0.clone(), - vec![SIGNER.1].into_iter().collect(), - TAP_EIP712_DOMAIN_SEPARATOR.clone(), - 100 * 1024, - 100 * 1024, - 1, - ) - .await - .unwrap(); -} - -async fn create_mock_server() { - // Start a mock graphql server using wiremock - let mock_server = MockServer::start().await; - - // Mock result for TAP redeem txs for (allocation, sender) pair. - mock_server - .register( - Mock::given(method("POST")) - .and(body_string_contains("transactions")) - .respond_with( - ResponseTemplate::new(200) - .set_body_json(json!({ "data": { "transactions": []}})), - ), - ) - .await; -} - -async fn create_sender_accounts_manager() {} - -#[sqlx::test(migrations = "../migrations")] -async fn test_full_happy_path(pgpool: PgPool) { - - // create mock server - - // create mock server for getting escrow subgraph - - // create sender_allocation_manager - - // add sender account - - // add sender allocation 1 - // add sender allocation 2 - // add sender allocation 3 - - // store receipt - - // check if receipt was detected by notification handler - - // check if it was added on the sender_allocation - - // check if it was added on the sender_account - - // add more receipts on both until a trigger value is reached - - // check if it was triggered - - // check if there's a rav on the database - - // close the sender allocation 1 - - // check if the rav 1 is now final - - // close the sender account - - // check if the rav 2 is now final - - // kill the manager (simulating a shutdown of the system) - - // check if the rav 3 is still non-final -} From fc3097cf7236b6ea03496564da83afd47eff70cc Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Thu, 11 Apr 2024 14:22:06 -0300 Subject: [PATCH 26/41] style: cargo clippy --- tap-agent/src/agent/sender_account.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/tap-agent/src/agent/sender_account.rs b/tap-agent/src/agent/sender_account.rs index f88f37fd7..34fa6ecdc 100644 --- a/tap-agent/src/agent/sender_account.rs +++ b/tap-agent/src/agent/sender_account.rs @@ -405,13 +405,10 @@ mod tests { message: Self::Msg, _state: &mut Self::State, ) -> Result<(), ActorProcessingErr> { - match message { - SenderAllocationMessage::TriggerRAVRequest(reply) => { - self.triggered_rav_request - .store(true, std::sync::atomic::Ordering::SeqCst); - reply.send(UnaggregatedReceipts::default())?; - } - _ => {} + if let SenderAllocationMessage::TriggerRAVRequest(reply) = message { + self.triggered_rav_request + .store(true, std::sync::atomic::Ordering::SeqCst); + reply.send(UnaggregatedReceipts::default())?; } Ok(()) } From 791f1ee97865b5b7bb2aa4c357e07bcd6470665b Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Thu, 11 Apr 2024 16:31:58 -0300 Subject: [PATCH 27/41] refactor: update messages and add receipt notification test --- tap-agent/src/agent/sender_account.rs | 176 ++++++++----- .../src/agent/sender_accounts_manager.rs | 238 +++++++++++++----- tap-agent/src/agent/sender_allocation.rs | 53 ++-- 3 files changed, 307 insertions(+), 160 deletions(-) diff --git a/tap-agent/src/agent/sender_account.rs b/tap-agent/src/agent/sender_account.rs index 34fa6ecdc..ca82b86ae 100644 --- a/tap-agent/src/agent/sender_account.rs +++ b/tap-agent/src/agent/sender_account.rs @@ -23,9 +23,7 @@ use crate::{ #[derive(Debug)] pub enum SenderAccountMessage { - CreateSenderAllocation(Address), UpdateAllocationIds(HashSet
), - RemoveSenderAccount, UpdateReceiptFees(Address, UnaggregatedReceipts), #[cfg(test)] GetAllocationTracker(ractor::RpcReplyPort), @@ -73,6 +71,33 @@ pub struct State { } impl State { + async fn create_sender_allocation( + &self, + sender_account_ref: ActorRef, + allocation_id: Address, + ) -> Result<()> { + let args = SenderAllocationArgs { + config: self.config, + pgpool: self.pgpool.clone(), + allocation_id, + sender: self.sender, + escrow_accounts: self.escrow_accounts.clone(), + escrow_subgraph: self.escrow_subgraph, + escrow_adapter: self.escrow_adapter.clone(), + domain_separator: self.domain_separator.clone(), + sender_aggregator_endpoint: self.sender_aggregator_endpoint.clone(), + sender_account_ref: sender_account_ref.clone(), + }; + + SenderAllocation::spawn_linked( + Some(self.format_sender_allocation(&allocation_id)), + SenderAllocation, + args, + sender_account_ref.get_cell(), + ) + .await?; + Ok(()) + } fn format_sender_allocation(&self, allocation_id: &Address) -> String { let mut sender_allocation_id = String::new(); if let Some(prefix) = &self.prefix { @@ -85,13 +110,13 @@ impl State { async fn rav_requester_single(&mut self) -> Result<()> { let Some(allocation_id) = self.allocation_id_tracker.get_heaviest_allocation_id() else { - anyhow::bail!("Error while getting allocation with most unaggregated fees"); + anyhow::bail!("Error while getting the heaviest allocation because none has unaggregated fees tracked"); }; let sender_allocation_id = self.format_sender_allocation(&allocation_id); let allocation = ActorRef::::where_is(sender_allocation_id); let Some(allocation) = allocation else { - anyhow::bail!("Error while getting allocation with most unaggregated fees"); + anyhow::bail!("Error while getting allocation actor with most unaggregated fees"); }; // we call and wait for the response so we don't process anymore update let result = call!(allocation, SenderAllocationMessage::TriggerRAVRequest)?; @@ -140,16 +165,11 @@ impl Actor for SenderAccount { } }); - for allocation_id in &allocation_ids { - // Create a sender allocation for each allocation - myself.cast(SenderAccountMessage::CreateSenderAllocation(*allocation_id))?; - } - let escrow_adapter = EscrowAdapter::new(escrow_accounts.clone(), sender_id); - Ok(State { + let state = State { allocation_id_tracker: AllocationIdTracker::default(), - allocation_ids, + allocation_ids: allocation_ids.clone(), _indexer_allocations_handle, prefix, escrow_accounts, @@ -160,7 +180,16 @@ impl Actor for SenderAccount { config, pgpool, sender: sender_id, - }) + }; + + for allocation_id in &allocation_ids { + // Create a sender allocation for each allocation + state + .create_sender_allocation(myself.clone(), *allocation_id) + .await?; + } + + Ok(state) } async fn handle( @@ -170,9 +199,6 @@ impl Actor for SenderAccount { state: &mut Self::State, ) -> std::result::Result<(), ActorProcessingErr> { match message { - SenderAccountMessage::RemoveSenderAccount => { - myself.stop(Some("Received Remove Sender Account message".into())) - } SenderAccountMessage::UpdateReceiptFees(allocation_id, unaggregated_fees) => { let tracker = &mut state.allocation_id_tracker; tracker.add_or_update(allocation_id, unaggregated_fees.value); @@ -181,32 +207,12 @@ impl Actor for SenderAccount { state.rav_requester_single().await?; } } - SenderAccountMessage::CreateSenderAllocation(allocation_id) => { - let args = SenderAllocationArgs { - config: state.config, - pgpool: state.pgpool.clone(), - allocation_id, - sender: state.sender, - escrow_accounts: state.escrow_accounts.clone(), - escrow_subgraph: state.escrow_subgraph, - escrow_adapter: state.escrow_adapter.clone(), - domain_separator: state.domain_separator.clone(), - sender_aggregator_endpoint: state.sender_aggregator_endpoint.clone(), - sender_account_ref: myself.clone(), - }; - - SenderAllocation::spawn_linked( - Some(state.format_sender_allocation(&allocation_id)), - SenderAllocation, - args, - myself.get_cell(), - ) - .await?; - } SenderAccountMessage::UpdateAllocationIds(allocation_ids) => { // Create new sender allocations for allocation_id in allocation_ids.difference(&state.allocation_ids) { - myself.cast(SenderAccountMessage::CreateSenderAllocation(*allocation_id))?; + state + .create_sender_allocation(myself.clone(), *allocation_id) + .await?; } // Remove sender allocations @@ -214,7 +220,7 @@ impl Actor for SenderAccount { if let Some(sender_handle) = ActorRef::::where_is( state.format_sender_allocation(allocation_id), ) { - sender_handle.cast(SenderAllocationMessage::CloseAllocation)?; + sender_handle.stop(None); } } @@ -236,11 +242,25 @@ impl Actor for SenderAccount { &self, _myself: ActorRef, message: SupervisionEvent, - _state: &mut Self::State, + state: &mut Self::State, ) -> std::result::Result<(), ActorProcessingErr> { match message { - SupervisionEvent::ActorTerminated(_, _, _) | SupervisionEvent::ActorPanicked(_, _) => { + SupervisionEvent::ActorTerminated(cell, _, _) + | SupervisionEvent::ActorPanicked(cell, _) => { // what to do in case of termination or panic? + + let Some(allocation_id) = cell.get_name() else { + return Ok(()); + }; + let Some(allocation_id) = allocation_id.split(':').last() else { + return Ok(()); + }; + let Ok(allocation_id) = Address::parse_checksummed(allocation_id, None) else { + return Ok(()); + }; + + let tracker = &mut state.allocation_id_tracker; + tracker.add_or_update(allocation_id, 0); } _ => {} } @@ -249,8 +269,9 @@ impl Actor for SenderAccount { } #[cfg(test)] -mod tests { +pub mod tests { use super::{SenderAccount, SenderAccountArgs, SenderAccountMessage}; + use crate::agent::sender_accounts_manager::NewReceiptNotification; use crate::agent::sender_allocation::SenderAllocationMessage; use crate::agent::unaggregated_receipts::UnaggregatedReceipts; use crate::config; @@ -266,7 +287,7 @@ mod tests { use sqlx::PgPool; use std::collections::{HashMap, HashSet}; use std::sync::atomic::{AtomicBool, AtomicU32}; - use std::sync::Arc; + use std::sync::{Arc, Mutex}; use std::time::Duration; // we implement the PartialEq and Eq traits for SenderAccountMessage to be able to compare @@ -275,7 +296,6 @@ mod tests { impl PartialEq for SenderAccountMessage { fn eq(&self, other: &Self) -> bool { match (self, other) { - (Self::CreateSenderAllocation(l0), Self::CreateSenderAllocation(r0)) => l0 == r0, (Self::UpdateAllocationIds(l0), Self::UpdateAllocationIds(r0)) => l0 == r0, (Self::UpdateReceiptFees(l0, l1), Self::UpdateReceiptFees(r0, r1)) => { l0 == r0 && l1 == r1 @@ -287,7 +307,6 @@ mod tests { static PREFIX_ID: AtomicU32 = AtomicU32::new(0); const DUMMY_URL: &str = "http://localhost:1234"; - const VALUE_PER_RECEIPT: u128 = 100; const TRIGGER_VALUE: u128 = 500; async fn create_sender_account( @@ -381,8 +400,33 @@ mod tests { handle.await.unwrap(); } - struct MockSenderAllocation { + pub struct MockSenderAllocation { triggered_rav_request: Arc, + receipts: Arc>>, + } + + impl MockSenderAllocation { + pub fn new_with_triggered_rav_request() -> (Self, Arc) { + let triggered_rav_request = Arc::new(AtomicBool::new(false)); + ( + Self { + triggered_rav_request: triggered_rav_request.clone(), + receipts: Arc::new(Mutex::new(Vec::new())), + }, + triggered_rav_request, + ) + } + + pub fn new_with_receipts() -> (Self, Arc>>) { + let receipts = Arc::new(Mutex::new(Vec::new())); + ( + Self { + triggered_rav_request: Arc::new(AtomicBool::new(false)), + receipts: receipts.clone(), + }, + receipts, + ) + } } #[async_trait::async_trait] @@ -405,10 +449,16 @@ mod tests { message: Self::Msg, _state: &mut Self::State, ) -> Result<(), ActorProcessingErr> { - if let SenderAllocationMessage::TriggerRAVRequest(reply) = message { - self.triggered_rav_request - .store(true, std::sync::atomic::Ordering::SeqCst); - reply.send(UnaggregatedReceipts::default())?; + match message { + SenderAllocationMessage::TriggerRAVRequest(reply) => { + self.triggered_rav_request + .store(true, std::sync::atomic::Ordering::SeqCst); + reply.send(UnaggregatedReceipts::default())?; + } + SenderAllocationMessage::NewReceipt(receipt) => { + self.receipts.lock().unwrap().push(receipt); + } + _ => {} } Ok(()) } @@ -423,18 +473,14 @@ mod tests { ActorRef, JoinHandle<()>, ) { - let triggered_rav_request = Arc::new(AtomicBool::new(false)); + let (mock_sender_allocation, triggered_rav_request) = + MockSenderAllocation::new_with_triggered_rav_request(); let name = format!("{}:{}:{}", prefix, sender, allocation); - let (sender_account, join_handle) = MockSenderAllocation::spawn( - Some(name), - MockSenderAllocation { - triggered_rav_request: triggered_rav_request.clone(), - }, - (), - ) - .await - .unwrap(); + let (sender_account, join_handle) = + MockSenderAllocation::spawn(Some(name), mock_sender_allocation, ()) + .await + .unwrap(); (triggered_rav_request, sender_account, join_handle) } @@ -450,7 +496,7 @@ mod tests { .cast(SenderAccountMessage::UpdateReceiptFees( *ALLOCATION_ID_0, UnaggregatedReceipts { - value: VALUE_PER_RECEIPT, + value: TRIGGER_VALUE - 1, last_id: 10, }, )) @@ -510,15 +556,13 @@ mod tests { }; // stop - sender_account - .cast(SenderAccountMessage::RemoveSenderAccount) - .unwrap(); - tokio::time::sleep(Duration::from_millis(10)).await; + sender_account.stop_and_wait(None, None).await.unwrap(); // check if sender_account is stopped - assert_eq!(sender_account.get_status(), ActorStatus::Stopped); + tokio::time::sleep(Duration::from_millis(10)).await; + // check if sender_allocation is also stopped assert_eq!(sender_allocation.get_status(), ActorStatus::Stopped); diff --git a/tap-agent/src/agent/sender_accounts_manager.rs b/tap-agent/src/agent/sender_accounts_manager.rs index 508fd4224..d96b4da98 100644 --- a/tap-agent/src/agent/sender_accounts_manager.rs +++ b/tap-agent/src/agent/sender_accounts_manager.rs @@ -11,7 +11,7 @@ use anyhow::Result; use eventuals::{Eventual, EventualExt, PipeHandle}; use indexer_common::escrow_accounts::EscrowAccounts; use indexer_common::prelude::{Allocation, SubgraphClient}; -use ractor::{Actor, ActorProcessingErr, ActorRef, SupervisionEvent}; +use ractor::{Actor, ActorCell, ActorProcessingErr, ActorRef, SupervisionEvent}; use serde::Deserialize; use sqlx::{postgres::PgListener, PgPool}; use thegraph::types::Address; @@ -34,7 +34,6 @@ pub struct SenderAccountsManager; pub enum SenderAccountsManagerMessage { UpdateSenderAccounts(HashSet
), - CreateSenderAccount(Address, HashSet
), } pub struct SenderAccountsManagerArgs { @@ -97,8 +96,11 @@ impl Actor for SenderAccountsManager { 'scalar_tap_receipt_notification'", ); // Start the new_receipts_watcher task that will consume from the `pglistener` - let new_receipts_watcher_handle = - tokio::spawn(new_receipts_watcher(pglistener, escrow_accounts.clone())); + let new_receipts_watcher_handle = tokio::spawn(new_receipts_watcher( + pglistener, + escrow_accounts.clone(), + prefix.clone(), + )); let clone = myself.clone(); let _eligible_allocations_senders_pipe = escrow_accounts.clone().pipe_async(move |escrow_accounts| { @@ -136,10 +138,9 @@ impl Actor for SenderAccountsManager { }; for (sender_id, allocation_ids) in sender_allocation { - myself.cast(SenderAccountsManagerMessage::CreateSenderAccount( - sender_id, - allocation_ids, - ))?; + state + .create_sender_account(myself.get_cell(), sender_id, allocation_ids) + .await?; } Ok(state) @@ -165,10 +166,9 @@ impl Actor for SenderAccountsManager { SenderAccountsManagerMessage::UpdateSenderAccounts(target_senders) => { // Create new sender accounts for sender in target_senders.difference(&state.sender_ids) { - myself.cast(SenderAccountsManagerMessage::CreateSenderAccount( - *sender, - HashSet::new(), - ))?; + state + .create_sender_account(myself.get_cell(), *sender, HashSet::new()) + .await?; } // Remove sender accounts @@ -176,22 +176,12 @@ impl Actor for SenderAccountsManager { if let Some(sender_handle) = ActorRef::::where_is( state.format_sender_account(sender), ) { - sender_handle.cast(SenderAccountMessage::RemoveSenderAccount)?; + sender_handle.stop(None); } } state.sender_ids = target_senders; } - SenderAccountsManagerMessage::CreateSenderAccount(sender_id, allocation_ids) => { - let args = state.new_sender_account_args(&sender_id, allocation_ids)?; - SenderAccount::spawn_linked( - Some(state.format_sender_account(&sender_id)), - SenderAccount, - args, - myself.get_cell(), - ) - .await?; - } } Ok(()) } @@ -225,6 +215,23 @@ impl State { sender_allocation_id } + async fn create_sender_account( + &self, + supervisor: ActorCell, + sender_id: Address, + allocation_ids: HashSet
, + ) -> anyhow::Result<()> { + let args = self.new_sender_account_args(&sender_id, allocation_ids)?; + SenderAccount::spawn_linked( + Some(self.format_sender_account(&sender_id)), + SenderAccount, + args, + supervisor, + ) + .await?; + Ok(()) + } + async fn get_pending_sender_allocation_id(&self) -> HashMap> { let escrow_accounts_snapshot = self .escrow_accounts @@ -357,6 +364,7 @@ impl State { async fn new_receipts_watcher( mut pglistener: PgListener, escrow_accounts: Eventual, + prefix: Option, ) { loop { // TODO: recover from errors or shutdown the whole program? @@ -370,28 +378,28 @@ async fn new_receipts_watcher( NewReceiptNotification", ); - let sender_address = escrow_accounts + let Ok(sender_address) = escrow_accounts .value() .await .expect("should be able to get escrow accounts") - .get_sender_for_signer(&new_receipt_notification.signer_address); - - let sender_address = match sender_address { - Ok(sender_address) => sender_address, - Err(_) => { - error!( - "No sender address found for receipt signer address {}. \ + .get_sender_for_signer(&new_receipt_notification.signer_address) + else { + error!( + "No sender address found for receipt signer address {}. \ This should not happen.", - new_receipt_notification.signer_address - ); - // TODO: save the receipt in the failed receipts table? - continue; - } + new_receipt_notification.signer_address + ); + // TODO: save the receipt in the failed receipts table? + continue; }; + let allocation_id = &new_receipt_notification.allocation_id; if let Some(sender_allocation) = ActorRef::::where_is(format!( - "{sender_address}:{allocation_id}" + "{}{sender_address}:{allocation_id}", + prefix + .as_ref() + .map_or(String::default(), |prefix| format!("{prefix}:")) )) { if let Err(e) = sender_allocation.cast(SenderAllocationMessage::NewReceipt( new_receipt_notification, @@ -403,7 +411,7 @@ async fn new_receipts_watcher( } } else { warn!( - "No sender_allocation_manager found for sender_address {} to process new \ + "No sender_allocation found for sender_address {} to process new \ receipt notification. This should not happen.", sender_address ); @@ -414,8 +422,10 @@ async fn new_receipts_watcher( #[cfg(test)] mod tests { use super::{ - SenderAccountsManager, SenderAccountsManagerArgs, SenderAccountsManagerMessage, State, + new_receipts_watcher, SenderAccountsManager, SenderAccountsManagerArgs, + SenderAccountsManagerMessage, State, }; + use crate::agent::sender_account::tests::MockSenderAllocation; use crate::agent::sender_account::SenderAccountMessage; use crate::config; use crate::tap::test_utils::{ @@ -428,10 +438,12 @@ mod tests { use indexer_common::escrow_accounts::EscrowAccounts; use indexer_common::prelude::{DeploymentDetails, SubgraphClient}; use ractor::concurrency::JoinHandle; - use ractor::{Actor, ActorRef}; + use ractor::{Actor, ActorProcessingErr, ActorRef}; + use sqlx::postgres::PgListener; use sqlx::PgPool; use std::collections::{HashMap, HashSet}; use std::sync::atomic::AtomicU32; + use std::time::Duration; const DUMMY_URL: &str = "http://localhost:1234"; static PREFIX_ID: AtomicU32 = AtomicU32::new(0); @@ -508,24 +520,40 @@ mod tests { join_handle.await.unwrap(); } - #[sqlx::test(migrations = "../migrations")] - async fn test_pending_sender_allocations(pgpool: PgPool) { + fn create_state(pgpool: PgPool) -> (String, State) { let config = get_config(); let senders_to_signers = vec![(SENDER.1, vec![SIGNER.1])].into_iter().collect(); let escrow_accounts = EscrowAccounts::new(HashMap::new(), senders_to_signers); - let state = State { - config, - domain_separator: TAP_EIP712_DOMAIN_SEPARATOR.clone(), - sender_ids: HashSet::new(), - new_receipts_watcher_handle: tokio::spawn(async {}), - _eligible_allocations_senders_pipe: Eventual::from_value(()).pipe_async(|_| async {}), - pgpool: pgpool.clone(), - indexer_allocations: Eventual::from_value(HashSet::new()), - escrow_accounts: Eventual::from_value(escrow_accounts), - escrow_subgraph: get_subgraph_client(), - sender_aggregator_endpoints: HashMap::new(), - prefix: None, - }; + + let prefix = format!( + "test-{}", + PREFIX_ID.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + ); + ( + prefix.clone(), + State { + config, + domain_separator: TAP_EIP712_DOMAIN_SEPARATOR.clone(), + sender_ids: HashSet::new(), + new_receipts_watcher_handle: tokio::spawn(async {}), + _eligible_allocations_senders_pipe: Eventual::from_value(()) + .pipe_async(|_| async {}), + pgpool, + indexer_allocations: Eventual::from_value(HashSet::new()), + escrow_accounts: Eventual::from_value(escrow_accounts), + escrow_subgraph: get_subgraph_client(), + sender_aggregator_endpoints: HashMap::from([ + (SENDER.1, String::from("http://localhost:8000")), + (SENDER_2.1, String::from("http://localhost:8000")), + ]), + prefix: Some(prefix), + }, + ) + } + + #[sqlx::test(migrations = "../migrations")] + async fn test_pending_sender_allocations(pgpool: PgPool) { + let (_, state) = create_state(pgpool.clone()); // add receipts to the database for i in 1..=10 { @@ -583,21 +611,103 @@ mod tests { #[sqlx::test(migrations = "../migrations")] async fn test_create_sender_account(pgpool: PgPool) { - let (prefix, (actor, join_handle)) = create_sender_accounts_manager(pgpool).await; - actor - .cast(SenderAccountsManagerMessage::CreateSenderAccount( - SENDER_2.1, - HashSet::new(), - )) - .unwrap(); + struct DummyActor; + #[async_trait::async_trait] + impl Actor for DummyActor { + type Msg = (); + type State = (); + type Arguments = (); + + async fn pre_start( + &self, + _: ActorRef, + _: Self::Arguments, + ) -> Result { + Ok(()) + } + } + + let (prefix, state) = create_state(pgpool.clone()); + let (supervisor, handle) = DummyActor::spawn(None, DummyActor, ()).await.unwrap(); // we wait to check if the sender is created + + state + .create_sender_account(supervisor.get_cell(), SENDER_2.1, HashSet::new()) + .await + .unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(10)).await; let actor_ref = ActorRef::::where_is(format!("{}:{}", prefix, SENDER_2.1)); assert!(actor_ref.is_some()); - actor.stop_and_wait(None, None).await.unwrap(); - join_handle.await.unwrap(); + supervisor.stop_and_wait(None, None).await.unwrap(); + handle.await.unwrap(); + } + + #[sqlx::test(migrations = "../migrations")] + async fn test_receive_notifications_(pgpool: PgPool) { + let prefix = format!( + "test-{}", + PREFIX_ID.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + ); + // create dummy allocation + + let (mock_sender_allocation, receipts) = MockSenderAllocation::new_with_receipts(); + let _ = MockSenderAllocation::spawn( + Some(format!( + "{}:{}:{}", + prefix.clone(), + SENDER.1, + *ALLOCATION_ID_0 + )), + mock_sender_allocation, + (), + ) + .await + .unwrap(); + + // create tokio task to listen for notifications + + let mut pglistener = PgListener::connect_with(&pgpool.clone()).await.unwrap(); + pglistener + .listen("scalar_tap_receipt_notification") + .await + .expect( + "should be able to subscribe to Postgres Notify events on the channel \ + 'scalar_tap_receipt_notification'", + ); + + let escrow_accounts_eventual = Eventual::from_value(EscrowAccounts::new( + HashMap::from([(SENDER.1, 1000.into())]), + HashMap::from([(SENDER.1, vec![SIGNER.1])]), + )); + + // Start the new_receipts_watcher task that will consume from the `pglistener` + let new_receipts_watcher_handle = tokio::spawn(new_receipts_watcher( + pglistener, + escrow_accounts_eventual, + Some(prefix.clone()), + )); + + // add receipts to the database + for i in 1..=10 { + let receipt = create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i, i.into()); + store_receipt(&pgpool, receipt.signed_receipt()) + .await + .unwrap(); + } + + tokio::time::sleep(Duration::from_millis(100)).await; + + // check if receipt notification was sent to the allocation + let receipts = receipts.lock().unwrap(); + assert_eq!(receipts.len(), 10); + for (i, receipt) in receipts.iter().enumerate() { + assert_eq!((i + 1) as u64, receipt.id); + } + + new_receipts_watcher_handle.abort(); } } diff --git a/tap-agent/src/agent/sender_allocation.rs b/tap-agent/src/agent/sender_allocation.rs index 718e992f2..99831a157 100644 --- a/tap-agent/src/agent/sender_allocation.rs +++ b/tap-agent/src/agent/sender_allocation.rs @@ -68,7 +68,6 @@ pub struct SenderAllocationArgs { pub enum SenderAllocationMessage { NewReceipt(NewReceiptNotification), TriggerRAVRequest(RpcReplyPort), - CloseAllocation, #[cfg(test)] GetUnaggregatedReceipts(RpcReplyPort), } @@ -97,12 +96,29 @@ impl Actor for SenderAllocation { Ok(state) } + // this method only runs on graceful stop (real close allocation) + // if the actor crashes, this is not ran async fn post_stop( &self, _myself: ActorRef, state: &mut Self::State, ) -> std::result::Result<(), ActorProcessingErr> { - // cleanup receipt fees for sender + // Request a RAV and mark the allocation as final. + if state.unaggregated_fees.value > 0 { + state.rav_requester_single().await.inspect_err(|e| { + error!( + "Error while requesting RAV for sender {} and allocation {}: {}", + state.sender, state.allocation_id, e + ); + })?; + } + state.mark_rav_final().await.inspect_err(|e| { + error!( + "Error while marking allocation {} as final for sender {}: {}", + state.allocation_id, state.sender, e + ); + })?; + state .sender_account_ref .cast(SenderAccountMessage::UpdateReceiptFees( @@ -114,7 +130,7 @@ impl Actor for SenderAllocation { async fn handle( &self, - myself: ActorRef, + _myself: ActorRef, message: Self::Msg, state: &mut Self::State, ) -> std::result::Result<(), ActorProcessingErr> { @@ -161,27 +177,6 @@ impl Actor for SenderAllocation { let _ = reply.send(state.unaggregated_fees.clone()); } } - - SenderAllocationMessage::CloseAllocation => { - // Request a RAV and mark the allocation as final. - - if state.unaggregated_fees.value > 0 { - state.rav_requester_single().await.inspect_err(|e| { - error!( - "Error while requesting RAV for sender {} and allocation {}: {}", - state.sender, state.allocation_id, e - ); - })?; - } - state.mark_rav_final().await.inspect_err(|e| { - error!( - "Error while marking allocation {} as final for sender {}: {}", - state.allocation_id, state.sender, e - ); - })?; - // stop and trigger post_stop - myself.stop(None); - } #[cfg(test)] SenderAllocationMessage::GetUnaggregatedReceipts(reply) => { if !reply.is_closed() { @@ -839,10 +834,8 @@ mod tests { ) .await; - cast!(sender_allocation, SenderAllocationMessage::CloseAllocation).unwrap(); - - // should trigger rav request - tokio::time::sleep(std::time::Duration::from_millis(100)).await; + sender_allocation.stop_and_wait(None, None).await.unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(10)).await; // check if the actor is actually stopped assert_eq!(sender_allocation.get_status(), ActorStatus::Stopped); @@ -933,7 +926,7 @@ mod tests { ) .await; - cast!(sender_allocation, SenderAllocationMessage::CloseAllocation).unwrap(); + sender_allocation.stop_and_wait(None, None).await.unwrap(); // should trigger rav request await_trigger.notified().await; @@ -942,7 +935,7 @@ mod tests { // check if rav request is made assert!(aggregator_server.received_requests().await.is_some()); - tokio::time::sleep(std::time::Duration::from_millis(10)).await; + tokio::time::sleep(std::time::Duration::from_millis(100)).await; // check if the actor is actually stopped assert_eq!(sender_allocation.get_status(), ActorStatus::Stopped); From b24b4439d782ed960583d6a51e459f1c3d4e7762 Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Thu, 11 Apr 2024 19:12:31 -0300 Subject: [PATCH 28/41] test: use single prefix_id Signed-off-by: Gustavo Inacio --- tap-agent/src/agent/sender_account.rs | 2 +- tap-agent/src/agent/sender_accounts_manager.rs | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tap-agent/src/agent/sender_account.rs b/tap-agent/src/agent/sender_account.rs index ca82b86ae..0d5ff3723 100644 --- a/tap-agent/src/agent/sender_account.rs +++ b/tap-agent/src/agent/sender_account.rs @@ -305,7 +305,7 @@ pub mod tests { } } - static PREFIX_ID: AtomicU32 = AtomicU32::new(0); + pub static PREFIX_ID: AtomicU32 = AtomicU32::new(0); const DUMMY_URL: &str = "http://localhost:1234"; const TRIGGER_VALUE: u128 = 500; diff --git a/tap-agent/src/agent/sender_accounts_manager.rs b/tap-agent/src/agent/sender_accounts_manager.rs index d96b4da98..2d6eb3b09 100644 --- a/tap-agent/src/agent/sender_accounts_manager.rs +++ b/tap-agent/src/agent/sender_accounts_manager.rs @@ -425,7 +425,7 @@ mod tests { new_receipts_watcher, SenderAccountsManager, SenderAccountsManagerArgs, SenderAccountsManagerMessage, State, }; - use crate::agent::sender_account::tests::MockSenderAllocation; + use crate::agent::sender_account::tests::{MockSenderAllocation, PREFIX_ID}; use crate::agent::sender_account::SenderAccountMessage; use crate::config; use crate::tap::test_utils::{ @@ -442,11 +442,9 @@ mod tests { use sqlx::postgres::PgListener; use sqlx::PgPool; use std::collections::{HashMap, HashSet}; - use std::sync::atomic::AtomicU32; use std::time::Duration; const DUMMY_URL: &str = "http://localhost:1234"; - static PREFIX_ID: AtomicU32 = AtomicU32::new(0); fn get_subgraph_client() -> &'static SubgraphClient { Box::leak(Box::new(SubgraphClient::new( From c27f7a27cba7ef47b4760749cc68a6b20c3bf16c Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Thu, 11 Apr 2024 19:12:58 -0300 Subject: [PATCH 29/41] chore: fix rebase conflicts Signed-off-by: Gustavo Inacio --- tap-agent/src/agent/sender_allocation.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tap-agent/src/agent/sender_allocation.rs b/tap-agent/src/agent/sender_allocation.rs index 99831a157..efea868d5 100644 --- a/tap-agent/src/agent/sender_allocation.rs +++ b/tap-agent/src/agent/sender_allocation.rs @@ -112,7 +112,7 @@ impl Actor for SenderAllocation { ); })?; } - state.mark_rav_final().await.inspect_err(|e| { + state.mark_rav_last().await.inspect_err(|e| { error!( "Error while marking allocation {} as final for sender {}: {}", state.allocation_id, state.sender, e @@ -410,7 +410,7 @@ impl SenderAllocationState { // in case no rav was marked as final 0 => { warn!( - "No RAVs were updated as final for allocation {} and sender {}.", + "No RAVs were updated as last for allocation {} and sender {}.", self.allocation_id, self.sender ); Ok(()) @@ -1060,7 +1060,7 @@ mod tests { } #[sqlx::test(migrations = "../migrations")] - async fn test_mark_rav_final(pgpool: PgPool) { + async fn test_mark_rav_last(pgpool: PgPool) { let signed_rav = create_rav(*ALLOCATION_ID_0, SIGNER.0.clone(), 4, 10); store_rav(&pgpool, signed_rav, SENDER.1).await.unwrap(); @@ -1070,7 +1070,7 @@ mod tests { let state = SenderAllocationState::new(args); // mark rav as final - let result = state.mark_rav_final().await; + let result = state.mark_rav_last().await; // check if it fails assert!(result.is_ok()); From b85c1f0f206b07cebdaf353d39d2f7d75aa2b194 Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Thu, 11 Apr 2024 20:31:25 -0300 Subject: [PATCH 30/41] feat: add logging to agents Signed-off-by: Gustavo Inacio --- tap-agent/src/agent/sender_account.rs | 8 +++++ .../src/agent/sender_accounts_manager.rs | 29 +++++++++++++++---- tap-agent/src/agent/sender_allocation.rs | 24 +++++++++++++++ 3 files changed, 55 insertions(+), 6 deletions(-) diff --git a/tap-agent/src/agent/sender_account.rs b/tap-agent/src/agent/sender_account.rs index 0d5ff3723..eb36939e0 100644 --- a/tap-agent/src/agent/sender_account.rs +++ b/tap-agent/src/agent/sender_account.rs @@ -189,6 +189,7 @@ impl Actor for SenderAccount { .await?; } + tracing::info!(sender = %sender_id, "SenderAccount created!"); Ok(state) } @@ -198,6 +199,11 @@ impl Actor for SenderAccount { message: Self::Msg, state: &mut Self::State, ) -> std::result::Result<(), ActorProcessingErr> { + tracing::trace!( + sender = %state.sender, + message = ?message, + "New SenderAccount message" + ); match message { SenderAccountMessage::UpdateReceiptFees(allocation_id, unaggregated_fees) => { let tracker = &mut state.allocation_id_tracker; @@ -248,6 +254,8 @@ impl Actor for SenderAccount { SupervisionEvent::ActorTerminated(cell, _, _) | SupervisionEvent::ActorPanicked(cell, _) => { // what to do in case of termination or panic? + let sender_allocation = cell.get_name(); + tracing::warn!(?sender_allocation, "Actor SenderAllocation was terminated"); let Some(allocation_id) = cell.get_name() else { return Ok(()); diff --git a/tap-agent/src/agent/sender_accounts_manager.rs b/tap-agent/src/agent/sender_accounts_manager.rs index 2d6eb3b09..0e7b4f274 100644 --- a/tap-agent/src/agent/sender_accounts_manager.rs +++ b/tap-agent/src/agent/sender_accounts_manager.rs @@ -32,6 +32,7 @@ pub struct NewReceiptNotification { pub struct SenderAccountsManager; +#[derive(Debug)] pub enum SenderAccountsManagerMessage { UpdateSenderAccounts(HashSet
), } @@ -143,6 +144,7 @@ impl Actor for SenderAccountsManager { .await?; } + tracing::info!("SenderAccountManager created!"); Ok(state) } async fn post_stop( @@ -162,6 +164,11 @@ impl Actor for SenderAccountsManager { msg: Self::Msg, state: &mut Self::State, ) -> std::result::Result<(), ActorProcessingErr> { + tracing::trace!( + message = ?msg, + "New SenderAccountManager message" + ); + match msg { SenderAccountsManagerMessage::UpdateSenderAccounts(target_senders) => { // Create new sender accounts @@ -195,8 +202,10 @@ impl Actor for SenderAccountsManager { _state: &mut Self::State, ) -> std::result::Result<(), ActorProcessingErr> { match message { - SupervisionEvent::ActorTerminated(_, _, _) | SupervisionEvent::ActorPanicked(_, _) => { - // what to do in case of termination or panic + SupervisionEvent::ActorTerminated(cell, _, _) + | SupervisionEvent::ActorPanicked(cell, _) => { + let sender_id = cell.get_name(); + tracing::warn!(?sender_id, "Actor SenderAccount was terminated") } _ => {} } @@ -378,6 +387,11 @@ async fn new_receipts_watcher( NewReceiptNotification", ); + tracing::debug!( + notification = ?new_receipt_notification, + "New receipt notification detected!" + ); + let Ok(sender_address) = escrow_accounts .value() .await @@ -395,12 +409,14 @@ async fn new_receipts_watcher( let allocation_id = &new_receipt_notification.allocation_id; - if let Some(sender_allocation) = ActorRef::::where_is(format!( + let actor_name = format!( "{}{sender_address}:{allocation_id}", prefix .as_ref() .map_or(String::default(), |prefix| format!("{prefix}:")) - )) { + ); + + if let Some(sender_allocation) = ActorRef::::where_is(actor_name) { if let Err(e) = sender_allocation.cast(SenderAllocationMessage::NewReceipt( new_receipt_notification, )) { @@ -411,9 +427,10 @@ async fn new_receipts_watcher( } } else { warn!( - "No sender_allocation found for sender_address {} to process new \ + "No sender_allocation found for sender_address {}, allocation_id {} to process new \ receipt notification. This should not happen.", - sender_address + sender_address, + allocation_id ); } } diff --git a/tap-agent/src/agent/sender_allocation.rs b/tap-agent/src/agent/sender_allocation.rs index efea868d5..235e9a3ae 100644 --- a/tap-agent/src/agent/sender_allocation.rs +++ b/tap-agent/src/agent/sender_allocation.rs @@ -65,6 +65,7 @@ pub struct SenderAllocationArgs { pub sender_account_ref: ActorRef, } +#[derive(Debug)] pub enum SenderAllocationMessage { NewReceipt(NewReceiptNotification), TriggerRAVRequest(RpcReplyPort), @@ -93,6 +94,12 @@ impl Actor for SenderAllocation { state.unaggregated_fees.clone(), ))?; + tracing::info!( + sender = %state.sender, + allocation_id = %state.allocation_id, + "SenderAllocation created!", + ); + Ok(state) } @@ -103,6 +110,12 @@ impl Actor for SenderAllocation { _myself: ActorRef, state: &mut Self::State, ) -> std::result::Result<(), ActorProcessingErr> { + tracing::info!( + sender = %state.sender, + allocation_id = %state.allocation_id, + "Closing SenderAllocation, triggering last rav", + ); + // Request a RAV and mark the allocation as final. if state.unaggregated_fees.value > 0 { state.rav_requester_single().await.inspect_err(|e| { @@ -134,6 +147,12 @@ impl Actor for SenderAllocation { message: Self::Msg, state: &mut Self::State, ) -> std::result::Result<(), ActorProcessingErr> { + tracing::trace!( + sender = %state.sender, + allocation_id = %state.allocation_id, + ?message, + "New SenderAllocation message" + ); let unaggreated_fees = &mut state.unaggregated_fees; match message { SenderAllocationMessage::NewReceipt(NewReceiptNotification { @@ -394,6 +413,11 @@ impl SenderAllocationState { } pub async fn mark_rav_last(&self) -> Result<()> { + tracing::info!( + sender = %self.sender, + allocation_id = %self.allocation_id, + "Marking rav as last!", + ); let updated_rows = sqlx::query!( r#" UPDATE scalar_tap_ravs From f15b6f79fcc3e54697b1331af2a36a472f3b387d Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Thu, 11 Apr 2024 22:32:03 -0300 Subject: [PATCH 31/41] refactor: add actor recovery and logging --- tap-agent/src/agent/sender_account.rs | 32 +++++++++++-- .../src/agent/sender_accounts_manager.rs | 45 ++++++++++++++++--- 2 files changed, 69 insertions(+), 8 deletions(-) diff --git a/tap-agent/src/agent/sender_account.rs b/tap-agent/src/agent/sender_account.rs index eb36939e0..07a3bece7 100644 --- a/tap-agent/src/agent/sender_account.rs +++ b/tap-agent/src/agent/sender_account.rs @@ -246,30 +246,56 @@ impl Actor for SenderAccount { // is shutdown the supervisor on actor termination events async fn handle_supervisor_evt( &self, - _myself: ActorRef, + myself: ActorRef, message: SupervisionEvent, state: &mut Self::State, ) -> std::result::Result<(), ActorProcessingErr> { match message { - SupervisionEvent::ActorTerminated(cell, _, _) - | SupervisionEvent::ActorPanicked(cell, _) => { + SupervisionEvent::ActorTerminated(cell, _, _) => { // what to do in case of termination or panic? let sender_allocation = cell.get_name(); tracing::warn!(?sender_allocation, "Actor SenderAllocation was terminated"); let Some(allocation_id) = cell.get_name() else { + tracing::error!("SenderAllocation doesn't have a name"); return Ok(()); }; let Some(allocation_id) = allocation_id.split(':').last() else { + tracing::error!(%allocation_id, "Could not extract allocation_id from name"); return Ok(()); }; let Ok(allocation_id) = Address::parse_checksummed(allocation_id, None) else { + tracing::error!(%allocation_id, "Could not convert allocation_id to Address"); return Ok(()); }; let tracker = &mut state.allocation_id_tracker; tracker.add_or_update(allocation_id, 0); } + SupervisionEvent::ActorPanicked(cell, error) => { + let sender_allocation = cell.get_name(); + tracing::warn!( + ?sender_allocation, + ?error, + "Actor SenderAllocation panicked. Restarting..." + ); + let Some(allocation_id) = cell.get_name() else { + tracing::error!("SenderAllocation doesn't have a name"); + return Ok(()); + }; + let Some(allocation_id) = allocation_id.split(':').last() else { + tracing::error!(%allocation_id, "Could not extract allocation_id from name"); + return Ok(()); + }; + let Ok(allocation_id) = Address::parse_checksummed(allocation_id, None) else { + tracing::error!(%allocation_id, "Could not convert allocation_id to Address"); + return Ok(()); + }; + + state + .create_sender_allocation(myself.clone(), allocation_id) + .await?; + } _ => {} } Ok(()) diff --git a/tap-agent/src/agent/sender_accounts_manager.rs b/tap-agent/src/agent/sender_accounts_manager.rs index 0e7b4f274..94e37da5f 100644 --- a/tap-agent/src/agent/sender_accounts_manager.rs +++ b/tap-agent/src/agent/sender_accounts_manager.rs @@ -197,15 +197,50 @@ impl Actor for SenderAccountsManager { // is shutdown the supervisor on actor termination events async fn handle_supervisor_evt( &self, - _myself: ActorRef, + myself: ActorRef, message: SupervisionEvent, - _state: &mut Self::State, + state: &mut Self::State, ) -> std::result::Result<(), ActorProcessingErr> { match message { - SupervisionEvent::ActorTerminated(cell, _, _) - | SupervisionEvent::ActorPanicked(cell, _) => { + SupervisionEvent::ActorTerminated(cell, _, reason) => { + let sender_id = cell.get_name(); + tracing::warn!(?sender_id, ?reason, "Actor SenderAccount was terminated") + } + SupervisionEvent::ActorPanicked(cell, error) => { let sender_id = cell.get_name(); - tracing::warn!(?sender_id, "Actor SenderAccount was terminated") + tracing::warn!( + ?sender_id, + ?error, + "Actor SenderAccount panicked. Restarting..." + ); + let Some(sender_id) = cell.get_name() else { + tracing::error!("SenderAllocation doesn't have a name"); + return Ok(()); + }; + let Some(sender_id) = sender_id.split(':').last() else { + tracing::error!(%sender_id, "Could not extract sender_id from name"); + return Ok(()); + }; + let Ok(sender_id) = Address::parse_checksummed(sender_id, None) else { + tracing::error!(%sender_id, "Could not convert sender_id to Address"); + return Ok(()); + }; + + let mut sender_allocation = select! { + sender_allocation = state.get_pending_sender_allocation_id() => sender_allocation, + _ = tokio::time::sleep(std::time::Duration::from_secs(30)) => { + tracing::error!("Timeout while getting pending sender allocation ids"); + return Ok(()); + } + }; + + let allocations = sender_allocation + .remove(&sender_id) + .unwrap_or(HashSet::new()); + + state + .create_sender_account(myself.get_cell(), sender_id, allocations) + .await?; } _ => {} } From 3d4a6d2d277cc220f04b0c7c020fc0eca334d146 Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Thu, 11 Apr 2024 23:54:42 -0300 Subject: [PATCH 32/41] fix: add chain_id to network subgraph --- tap-agent/src/agent.rs | 3 ++- tap-agent/src/config.rs | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/tap-agent/src/agent.rs b/tap-agent/src/agent.rs index 40cbca000..c29272f17 100644 --- a/tap-agent/src/agent.rs +++ b/tap-agent/src/agent.rs @@ -36,6 +36,7 @@ pub async fn start_agent() -> ActorRef { network_subgraph_deployment, network_subgraph_endpoint, allocation_syncing_interval_ms, + chain_id, }, escrow_subgraph: EscrowSubgraph { @@ -72,7 +73,7 @@ pub async fn start_agent() -> ActorRef { let indexer_allocations = indexer_allocations( network_subgraph, *indexer_address, - 1, + *chain_id, Duration::from_millis(*allocation_syncing_interval_ms), ); diff --git a/tap-agent/src/config.rs b/tap-agent/src/config.rs index b614a2514..3b31c38e4 100644 --- a/tap-agent/src/config.rs +++ b/tap-agent/src/config.rs @@ -181,6 +181,14 @@ pub struct NetworkSubgraph { help = "Interval (in ms) for syncing indexer allocations from the network" )] pub allocation_syncing_interval_ms: u64, + + #[clap( + long, + value_name = "chain-id", + env = "NETWORK_CHAIN_ID", + help = "Interval (in ms) for syncing indexer allocations from the network" + )] + pub chain_id: u64, } #[derive(Clone, Debug, Args, Serialize, Deserialize, Default)] From 929dd1f8671d2869960bd25e82bd80f05f0cdaf4 Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Fri, 12 Apr 2024 10:09:21 -0300 Subject: [PATCH 33/41] fix: handle rav request error and more logs --- tap-agent/src/agent/sender_account.rs | 24 ++++++++++++++++++++++-- tap-agent/src/agent/sender_allocation.rs | 20 ++++++++++++-------- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/tap-agent/src/agent/sender_account.rs b/tap-agent/src/agent/sender_account.rs index 07a3bece7..ca5b98d86 100644 --- a/tap-agent/src/agent/sender_account.rs +++ b/tap-agent/src/agent/sender_account.rs @@ -10,7 +10,7 @@ use indexer_common::{escrow_accounts::EscrowAccounts, prelude::SubgraphClient}; use ractor::{call, Actor, ActorProcessingErr, ActorRef, SupervisionEvent}; use sqlx::PgPool; use thegraph::types::Address; -use tracing::error; +use tracing::{error, Level}; use super::sender_allocation::{SenderAllocation, SenderAllocationArgs}; use crate::agent::allocation_id_tracker::AllocationIdTracker; @@ -76,6 +76,11 @@ impl State { sender_account_ref: ActorRef, allocation_id: Address, ) -> Result<()> { + tracing::trace!( + %self.sender, + %allocation_id, + "SenderAccount is creating allocation." + ); let args = SenderAllocationArgs { config: self.config, pgpool: self.pgpool.clone(), @@ -199,8 +204,12 @@ impl Actor for SenderAccount { message: Self::Msg, state: &mut Self::State, ) -> std::result::Result<(), ActorProcessingErr> { - tracing::trace!( + tracing::span!( + Level::TRACE, + "SenderAccount handle()", sender = %state.sender, + ); + tracing::trace!( message = ?message, "New SenderAccount message" ); @@ -210,6 +219,11 @@ impl Actor for SenderAccount { tracker.add_or_update(allocation_id, unaggregated_fees.value); if tracker.get_total_fee() >= state.config.tap.rav_request_trigger_value.into() { + tracing::debug!( + total_fee = tracker.get_total_fee(), + trigger_value = state.config.tap.rav_request_trigger_value, + "Total fee greater than the trigger value. Triggering RAV request" + ); state.rav_requester_single().await?; } } @@ -226,10 +240,16 @@ impl Actor for SenderAccount { if let Some(sender_handle) = ActorRef::::where_is( state.format_sender_allocation(allocation_id), ) { + tracing::trace!(%allocation_id, "SenderAccount shutting down SenderAllocation"); sender_handle.stop(None); } } + tracing::trace!( + old_ids= ?state.allocation_ids, + new_ids = ?allocation_ids, + "Updating allocation ids" + ); state.allocation_ids = allocation_ids; } #[cfg(test)] diff --git a/tap-agent/src/agent/sender_allocation.rs b/tap-agent/src/agent/sender_allocation.rs index 235e9a3ae..438bd5762 100644 --- a/tap-agent/src/agent/sender_allocation.rs +++ b/tap-agent/src/agent/sender_allocation.rs @@ -182,15 +182,19 @@ impl Actor for SenderAllocation { // we use a blocking call here to ensure that only one RAV request is running at a time. SenderAllocationMessage::TriggerRAVRequest(reply) => { if state.unaggregated_fees.value > 0 { - state.rav_requester_single().await.map_err(|e| { - anyhow! { - "Error while requesting RAV for sender {} and allocation {}: {}", - state.sender, - state.allocation_id, - e + match state.rav_requester_single().await { + Ok(_) => { + state.unaggregated_fees = state.calculate_unaggregated_fee().await?; + } + Err(e) => { + error! ( + %state.sender, + %state.allocation_id, + error = %e, + "Error while requesting RAV ", + ); } - })?; - state.unaggregated_fees = state.calculate_unaggregated_fee().await?; + } } if !reply.is_closed() { let _ = reply.send(state.unaggregated_fees.clone()); From d63c557a9fec104e0b3e6621a5bbe9f547c491b1 Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Fri, 12 Apr 2024 11:07:07 -0300 Subject: [PATCH 34/41] chore: more logging --- tap-agent/src/agent/sender_account.rs | 6 ++++++ tap-agent/src/agent/sender_allocation.rs | 2 ++ 2 files changed, 8 insertions(+) diff --git a/tap-agent/src/agent/sender_account.rs b/tap-agent/src/agent/sender_account.rs index ca5b98d86..20d8a5404 100644 --- a/tap-agent/src/agent/sender_account.rs +++ b/tap-agent/src/agent/sender_account.rs @@ -270,6 +270,12 @@ impl Actor for SenderAccount { message: SupervisionEvent, state: &mut Self::State, ) -> std::result::Result<(), ActorProcessingErr> { + tracing::trace!( + sender = %state.sender, + message = ?message, + "New SenderAccount supervision event" + ); + match message { SupervisionEvent::ActorTerminated(cell, _, _) => { // what to do in case of termination or panic? diff --git a/tap-agent/src/agent/sender_allocation.rs b/tap-agent/src/agent/sender_allocation.rs index 438bd5762..2ab4a7da3 100644 --- a/tap-agent/src/agent/sender_allocation.rs +++ b/tap-agent/src/agent/sender_allocation.rs @@ -268,6 +268,7 @@ impl SenderAllocationState { /// Delete obsolete receipts in the DB w.r.t. the last RAV in DB, then update the tap manager /// with the latest unaggregated fees from the database. async fn calculate_unaggregated_fee(&self) -> Result { + tracing::trace!("calculate_unaggregated_fee()"); self.tap_manager.remove_obsolete_receipts().await?; let signers = signers_trimmed(&self.escrow_accounts, self.sender).await?; @@ -329,6 +330,7 @@ impl SenderAllocationState { /// Request a RAV from the sender's TAP aggregator. Only one RAV request will be running at a /// time through the use of an internal guard. async fn rav_requester_single(&self) -> Result<()> { + tracing::trace!("rav_requester_single()"); let RAVRequest { valid_receipts, previous_rav, From 8bf0566c9db0f030264223b6d5b75dc703e4debb Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Fri, 12 Apr 2024 12:55:17 -0300 Subject: [PATCH 35/41] fix: update sender_ids list for manager --- tap-agent/src/agent/sender_accounts_manager.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tap-agent/src/agent/sender_accounts_manager.rs b/tap-agent/src/agent/sender_accounts_manager.rs index 94e37da5f..673e7f4d5 100644 --- a/tap-agent/src/agent/sender_accounts_manager.rs +++ b/tap-agent/src/agent/sender_accounts_manager.rs @@ -118,7 +118,7 @@ impl Actor for SenderAccountsManager { } }); - let state = State { + let mut state = State { config, domain_separator, sender_ids: HashSet::new(), @@ -139,6 +139,7 @@ impl Actor for SenderAccountsManager { }; for (sender_id, allocation_ids) in sender_allocation { + state.sender_ids.insert(sender_id); state .create_sender_account(myself.get_cell(), sender_id, allocation_ids) .await?; From f759af6dc05b6913071b8d40a0cf323691bd8cb1 Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Fri, 12 Apr 2024 12:58:31 -0300 Subject: [PATCH 36/41] refactor: gracefully shutdown in case of manager stop --- tap-agent/src/agent.rs | 8 ++++---- tap-agent/src/main.rs | 16 ++++++++++------ 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/tap-agent/src/agent.rs b/tap-agent/src/agent.rs index c29272f17..e5360f27f 100644 --- a/tap-agent/src/agent.rs +++ b/tap-agent/src/agent.rs @@ -6,6 +6,7 @@ use std::time::Duration; use indexer_common::prelude::{ escrow_accounts, indexer_allocations, DeploymentDetails, SubgraphClient, }; +use ractor::concurrency::JoinHandle; use ractor::{Actor, ActorRef}; use crate::agent::sender_accounts_manager::{ @@ -21,7 +22,7 @@ pub mod sender_accounts_manager; pub mod sender_allocation; pub mod unaggregated_receipts; -pub async fn start_agent() -> ActorRef { +pub async fn start_agent() -> (ActorRef, JoinHandle<()>) { let Cli { ethereum: Ethereum { indexer_address }, indexer_infrastructure: @@ -115,8 +116,7 @@ pub async fn start_agent() -> ActorRef { prefix: None, }; - let (manager, _) = SenderAccountsManager::spawn(None, SenderAccountsManager, args) + SenderAccountsManager::spawn(None, SenderAccountsManager, args) .await - .expect("Failed to start sender accounts manager actor."); - manager + .expect("Failed to start sender accounts manager actor.") } diff --git a/tap-agent/src/main.rs b/tap-agent/src/main.rs index ab2cc7cfd..b53b13a32 100644 --- a/tap-agent/src/main.rs +++ b/tap-agent/src/main.rs @@ -2,8 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 use anyhow::Result; +use ractor::ActorStatus; use tokio::signal::unix::{signal, SignalKind}; -use tracing::{debug, info}; +use tracing::{debug, error, info}; use indexer_tap_agent::{agent, CONFIG}; @@ -13,13 +14,14 @@ async fn main() -> Result<()> { lazy_static::initialize(&CONFIG); debug!("Config: {:?}", *CONFIG); - let manager = agent::start_agent().await; + let (manager, handler) = agent::start_agent().await; info!("TAP Agent started."); // Have tokio wait for SIGTERM or SIGINT. let mut signal_sigint = signal(SignalKind::interrupt())?; let mut signal_sigterm = signal(SignalKind::terminate())?; tokio::select! { + _ = handler => error!("SenderAccountsManager stopped"), _ = signal_sigint.recv() => debug!("Received SIGINT."), _ = signal_sigterm.recv() => debug!("Received SIGTERM."), } @@ -27,10 +29,12 @@ async fn main() -> Result<()> { info!("Shutting down..."); // We don't want our actor to run any shutdown logic, so we kill it. - manager - .kill_and_wait(None) - .await - .expect("Failed to kill manager."); + if manager.get_status() == ActorStatus::Running { + manager + .kill_and_wait(None) + .await + .expect("Failed to kill manager."); + } // Stop the server and wait for it to finish gracefully. debug!("Goodbye!"); From 21723f02d1ba09a59bc973751429a91990dc2e6c Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Fri, 12 Apr 2024 15:06:37 -0300 Subject: [PATCH 37/41] fix: add chain_id default to 1 --- tap-agent/src/config.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/tap-agent/src/config.rs b/tap-agent/src/config.rs index 3b31c38e4..91ba8fa67 100644 --- a/tap-agent/src/config.rs +++ b/tap-agent/src/config.rs @@ -186,6 +186,7 @@ pub struct NetworkSubgraph { long, value_name = "chain-id", env = "NETWORK_CHAIN_ID", + default_value_t = 1, help = "Interval (in ms) for syncing indexer allocations from the network" )] pub chain_id: u64, From 740d9e1b2a32567f1fcd17363a91d334e2c6c7fd Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Sun, 14 Apr 2024 16:17:24 -0300 Subject: [PATCH 38/41] refactor: accept suggestions from review Signed-off-by: Gustavo Inacio --- tap-agent/src/agent.rs | 6 +++-- tap-agent/src/agent/allocation_id_tracker.rs | 26 +++++++++---------- tap-agent/src/agent/sender_account.rs | 25 +++++++++--------- .../src/agent/sender_accounts_manager.rs | 2 +- tap-agent/src/agent/sender_allocation.rs | 6 ----- tap-agent/src/config.rs | 9 ------- 6 files changed, 30 insertions(+), 44 deletions(-) diff --git a/tap-agent/src/agent.rs b/tap-agent/src/agent.rs index e5360f27f..2ce4c7f2a 100644 --- a/tap-agent/src/agent.rs +++ b/tap-agent/src/agent.rs @@ -22,6 +22,9 @@ pub mod sender_accounts_manager; pub mod sender_allocation; pub mod unaggregated_receipts; +/// constant graph network used in subgraphs +const GRAPH_NETWORK_ID: u64 = 1; + pub async fn start_agent() -> (ActorRef, JoinHandle<()>) { let Cli { ethereum: Ethereum { indexer_address }, @@ -37,7 +40,6 @@ pub async fn start_agent() -> (ActorRef, JoinHandl network_subgraph_deployment, network_subgraph_endpoint, allocation_syncing_interval_ms, - chain_id, }, escrow_subgraph: EscrowSubgraph { @@ -74,7 +76,7 @@ pub async fn start_agent() -> (ActorRef, JoinHandl let indexer_allocations = indexer_allocations( network_subgraph, *indexer_address, - *chain_id, + GRAPH_NETWORK_ID, Duration::from_millis(*allocation_syncing_interval_ms), ); diff --git a/tap-agent/src/agent/allocation_id_tracker.rs b/tap-agent/src/agent/allocation_id_tracker.rs index 74515a602..2a2fdca99 100644 --- a/tap-agent/src/agent/allocation_id_tracker.rs +++ b/tap-agent/src/agent/allocation_id_tracker.rs @@ -5,14 +5,14 @@ use alloy_primitives::Address; use std::collections::{BTreeMap, HashMap}; #[derive(Debug, Clone, Default)] -pub struct AllocationIdTracker { +pub struct SenderFeeTracker { id_to_fee: HashMap, fee_to_count: BTreeMap, total_fee: u128, } -impl AllocationIdTracker { - pub fn add_or_update(&mut self, id: Address, fee: u128) { +impl SenderFeeTracker { + pub fn update(&mut self, id: Address, fee: u128) { if let Some(&old_fee) = self.id_to_fee.get(&id) { self.total_fee -= old_fee; *self.fee_to_count.get_mut(&old_fee).unwrap() -= 1; @@ -46,7 +46,7 @@ impl AllocationIdTracker { #[cfg(test)] mod tests { - use super::AllocationIdTracker; + use super::SenderFeeTracker; use std::str::FromStr; use thegraph::types::Address; @@ -59,39 +59,39 @@ mod tests { let allocation_id_2: Address = Address::from_str("0xcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd").unwrap(); - let mut tracker = AllocationIdTracker::default(); + let mut tracker = SenderFeeTracker::default(); assert_eq!(tracker.get_heaviest_allocation_id(), None); assert_eq!(tracker.get_total_fee(), 0); - tracker.add_or_update(allocation_id_0, 10); + tracker.update(allocation_id_0, 10); assert_eq!(tracker.get_heaviest_allocation_id(), Some(allocation_id_0)); assert_eq!(tracker.get_total_fee(), 10); - tracker.add_or_update(allocation_id_2, 20); + tracker.update(allocation_id_2, 20); assert_eq!(tracker.get_heaviest_allocation_id(), Some(allocation_id_2)); assert_eq!(tracker.get_total_fee(), 30); - tracker.add_or_update(allocation_id_1, 30); + tracker.update(allocation_id_1, 30); assert_eq!(tracker.get_heaviest_allocation_id(), Some(allocation_id_1)); assert_eq!(tracker.get_total_fee(), 60); - tracker.add_or_update(allocation_id_2, 10); + tracker.update(allocation_id_2, 10); assert_eq!(tracker.get_heaviest_allocation_id(), Some(allocation_id_1)); assert_eq!(tracker.get_total_fee(), 50); - tracker.add_or_update(allocation_id_2, 40); + tracker.update(allocation_id_2, 40); assert_eq!(tracker.get_heaviest_allocation_id(), Some(allocation_id_2)); assert_eq!(tracker.get_total_fee(), 80); - tracker.add_or_update(allocation_id_1, 0); + tracker.update(allocation_id_1, 0); assert_eq!(tracker.get_heaviest_allocation_id(), Some(allocation_id_2)); assert_eq!(tracker.get_total_fee(), 50); - tracker.add_or_update(allocation_id_2, 0); + tracker.update(allocation_id_2, 0); assert_eq!(tracker.get_heaviest_allocation_id(), Some(allocation_id_0)); assert_eq!(tracker.get_total_fee(), 10); - tracker.add_or_update(allocation_id_0, 0); + tracker.update(allocation_id_0, 0); assert_eq!(tracker.get_heaviest_allocation_id(), None); assert_eq!(tracker.get_total_fee(), 0); } diff --git a/tap-agent/src/agent/sender_account.rs b/tap-agent/src/agent/sender_account.rs index 20d8a5404..5be961bfc 100644 --- a/tap-agent/src/agent/sender_account.rs +++ b/tap-agent/src/agent/sender_account.rs @@ -13,7 +13,7 @@ use thegraph::types::Address; use tracing::{error, Level}; use super::sender_allocation::{SenderAllocation, SenderAllocationArgs}; -use crate::agent::allocation_id_tracker::AllocationIdTracker; +use crate::agent::allocation_id_tracker::SenderFeeTracker; use crate::agent::sender_allocation::SenderAllocationMessage; use crate::agent::unaggregated_receipts::UnaggregatedReceipts; use crate::{ @@ -26,7 +26,7 @@ pub enum SenderAccountMessage { UpdateAllocationIds(HashSet
), UpdateReceiptFees(Address, UnaggregatedReceipts), #[cfg(test)] - GetAllocationTracker(ractor::RpcReplyPort), + GetSenderFeeTracker(ractor::RpcReplyPort), } /// A SenderAccount manages the receipts accounting between the indexer and the sender across @@ -54,7 +54,7 @@ pub struct SenderAccountArgs { } pub struct State { prefix: Option, - allocation_id_tracker: AllocationIdTracker, + sender_fee_tracker: SenderFeeTracker, allocation_ids: HashSet
, _indexer_allocations_handle: PipeHandle, sender: Address, @@ -114,7 +114,7 @@ impl State { } async fn rav_requester_single(&mut self) -> Result<()> { - let Some(allocation_id) = self.allocation_id_tracker.get_heaviest_allocation_id() else { + let Some(allocation_id) = self.sender_fee_tracker.get_heaviest_allocation_id() else { anyhow::bail!("Error while getting the heaviest allocation because none has unaggregated fees tracked"); }; let sender_allocation_id = self.format_sender_allocation(&allocation_id); @@ -126,8 +126,7 @@ impl State { // we call and wait for the response so we don't process anymore update let result = call!(allocation, SenderAllocationMessage::TriggerRAVRequest)?; - self.allocation_id_tracker - .add_or_update(allocation_id, result.value); + self.sender_fee_tracker.update(allocation_id, result.value); Ok(()) } } @@ -173,7 +172,7 @@ impl Actor for SenderAccount { let escrow_adapter = EscrowAdapter::new(escrow_accounts.clone(), sender_id); let state = State { - allocation_id_tracker: AllocationIdTracker::default(), + sender_fee_tracker: SenderFeeTracker::default(), allocation_ids: allocation_ids.clone(), _indexer_allocations_handle, prefix, @@ -215,8 +214,8 @@ impl Actor for SenderAccount { ); match message { SenderAccountMessage::UpdateReceiptFees(allocation_id, unaggregated_fees) => { - let tracker = &mut state.allocation_id_tracker; - tracker.add_or_update(allocation_id, unaggregated_fees.value); + let tracker = &mut state.sender_fee_tracker; + tracker.update(allocation_id, unaggregated_fees.value); if tracker.get_total_fee() >= state.config.tap.rav_request_trigger_value.into() { tracing::debug!( @@ -253,9 +252,9 @@ impl Actor for SenderAccount { state.allocation_ids = allocation_ids; } #[cfg(test)] - SenderAccountMessage::GetAllocationTracker(reply) => { + SenderAccountMessage::GetSenderFeeTracker(reply) => { if !reply.is_closed() { - let _ = reply.send(state.allocation_id_tracker.clone()); + let _ = reply.send(state.sender_fee_tracker.clone()); } } } @@ -295,8 +294,8 @@ impl Actor for SenderAccount { return Ok(()); }; - let tracker = &mut state.allocation_id_tracker; - tracker.add_or_update(allocation_id, 0); + let tracker = &mut state.sender_fee_tracker; + tracker.update(allocation_id, 0); } SupervisionEvent::ActorPanicked(cell, error) => { let sender_allocation = cell.get_name(); diff --git a/tap-agent/src/agent/sender_accounts_manager.rs b/tap-agent/src/agent/sender_accounts_manager.rs index 673e7f4d5..a4d75b0bb 100644 --- a/tap-agent/src/agent/sender_accounts_manager.rs +++ b/tap-agent/src/agent/sender_accounts_manager.rs @@ -205,7 +205,7 @@ impl Actor for SenderAccountsManager { match message { SupervisionEvent::ActorTerminated(cell, _, reason) => { let sender_id = cell.get_name(); - tracing::warn!(?sender_id, ?reason, "Actor SenderAccount was terminated") + tracing::info!(?sender_id, ?reason, "Actor SenderAccount was terminated") } SupervisionEvent::ActorPanicked(cell, error) => { let sender_id = cell.get_name(); diff --git a/tap-agent/src/agent/sender_allocation.rs b/tap-agent/src/agent/sender_allocation.rs index 2ab4a7da3..07080305b 100644 --- a/tap-agent/src/agent/sender_allocation.rs +++ b/tap-agent/src/agent/sender_allocation.rs @@ -132,12 +132,6 @@ impl Actor for SenderAllocation { ); })?; - state - .sender_account_ref - .cast(SenderAccountMessage::UpdateReceiptFees( - state.allocation_id, - UnaggregatedReceipts::default(), - ))?; Ok(()) } diff --git a/tap-agent/src/config.rs b/tap-agent/src/config.rs index 91ba8fa67..b614a2514 100644 --- a/tap-agent/src/config.rs +++ b/tap-agent/src/config.rs @@ -181,15 +181,6 @@ pub struct NetworkSubgraph { help = "Interval (in ms) for syncing indexer allocations from the network" )] pub allocation_syncing_interval_ms: u64, - - #[clap( - long, - value_name = "chain-id", - env = "NETWORK_CHAIN_ID", - default_value_t = 1, - help = "Interval (in ms) for syncing indexer allocations from the network" - )] - pub chain_id: u64, } #[derive(Clone, Debug, Args, Serialize, Deserialize, Default)] From 1641cc98702bca27edf68e9daee06338e98ba558 Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Sun, 14 Apr 2024 17:23:32 -0300 Subject: [PATCH 39/41] refacotr: rename sender fee, simplify logic and add overflow check Signed-off-by: Gustavo Inacio --- tap-agent/src/agent.rs | 2 +- tap-agent/src/agent/sender_account.rs | 2 +- ...on_id_tracker.rs => sender_fee_tracker.rs} | 53 +++++++++++-------- 3 files changed, 34 insertions(+), 23 deletions(-) rename tap-agent/src/agent/{allocation_id_tracker.rs => sender_fee_tracker.rs} (68%) diff --git a/tap-agent/src/agent.rs b/tap-agent/src/agent.rs index 2ce4c7f2a..2e7dff338 100644 --- a/tap-agent/src/agent.rs +++ b/tap-agent/src/agent.rs @@ -16,10 +16,10 @@ use crate::config::{Cli, EscrowSubgraph, Ethereum, IndexerInfrastructure, Networ use crate::{aggregator_endpoints, database, CONFIG, EIP_712_DOMAIN}; use sender_accounts_manager::SenderAccountsManager; -pub mod allocation_id_tracker; pub mod sender_account; pub mod sender_accounts_manager; pub mod sender_allocation; +pub mod sender_fee_tracker; pub mod unaggregated_receipts; /// constant graph network used in subgraphs diff --git a/tap-agent/src/agent/sender_account.rs b/tap-agent/src/agent/sender_account.rs index 5be961bfc..3cf0fcfb6 100644 --- a/tap-agent/src/agent/sender_account.rs +++ b/tap-agent/src/agent/sender_account.rs @@ -13,8 +13,8 @@ use thegraph::types::Address; use tracing::{error, Level}; use super::sender_allocation::{SenderAllocation, SenderAllocationArgs}; -use crate::agent::allocation_id_tracker::SenderFeeTracker; use crate::agent::sender_allocation::SenderAllocationMessage; +use crate::agent::sender_fee_tracker::SenderFeeTracker; use crate::agent::unaggregated_receipts::UnaggregatedReceipts; use crate::{ config::{self}, diff --git a/tap-agent/src/agent/allocation_id_tracker.rs b/tap-agent/src/agent/sender_fee_tracker.rs similarity index 68% rename from tap-agent/src/agent/allocation_id_tracker.rs rename to tap-agent/src/agent/sender_fee_tracker.rs index 2a2fdca99..2ac5c1684 100644 --- a/tap-agent/src/agent/allocation_id_tracker.rs +++ b/tap-agent/src/agent/sender_fee_tracker.rs @@ -2,41 +2,52 @@ // SPDX-License-Identifier: Apache-2.0 use alloy_primitives::Address; -use std::collections::{BTreeMap, HashMap}; +use std::collections::HashMap; +use tracing::error; #[derive(Debug, Clone, Default)] pub struct SenderFeeTracker { id_to_fee: HashMap, - fee_to_count: BTreeMap, total_fee: u128, } impl SenderFeeTracker { pub fn update(&mut self, id: Address, fee: u128) { - if let Some(&old_fee) = self.id_to_fee.get(&id) { - self.total_fee -= old_fee; - *self.fee_to_count.get_mut(&old_fee).unwrap() -= 1; - if self.fee_to_count[&old_fee] == 0 { - self.fee_to_count.remove(&old_fee); - } - } - if fee > 0 { - self.id_to_fee.insert(id, fee); - self.total_fee += fee; - *self.fee_to_count.entry(fee).or_insert(0) += 1; - } else { - self.id_to_fee.remove(&id); + // insert or update, if update remove old fee from total + if let Some(old_fee) = self.id_to_fee.insert(id, fee) { + self.total_fee -= old_fee; + } + self.total_fee = self.total_fee.checked_add(fee).unwrap_or_else(|| { + // This should never happen, but if it does, we want to know about it. + error!( + "Overflow when adding receipt value {} to total fee {}. \ + Setting total fee to u128::MAX.", + fee, self.total_fee + ); + u128::MAX + }); + } else if let Some(old_fee) = self.id_to_fee.remove(&id) { + self.total_fee -= old_fee; } } pub fn get_heaviest_allocation_id(&self) -> Option
{ - self.fee_to_count.iter().next_back().and_then(|(&fee, _)| { - self.id_to_fee - .iter() - .find(|(_, &f)| f == fee) - .map(|(&id, _)| id) - }) + // just loop over and get the biggest fee + self.id_to_fee + .iter() + .fold(None, |acc: Option<(&Address, u128)>, (addr, fee)| { + if let Some((_, max_fee)) = acc { + if *fee > max_fee { + Some((addr, *fee)) + } else { + acc + } + } else { + Some((addr, *fee)) + } + }) + .map(|(&id, _)| id) } pub fn get_total_fee(&self) -> u128 { From 7effcb5c8fd6bbc2d9c24a72c27b1ebc541fe6dc Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Sun, 14 Apr 2024 17:27:19 -0300 Subject: [PATCH 40/41] fix: remove check about last message, now using on sender account side Signed-off-by: Gustavo Inacio --- ...f29bed709159e76d5d6ecd4370ac10d521cff.json | 30 ----------- ...0bba5c3d01bc2106abe5839f1801d1683b8f6.json | 30 +++++++++++ tap-agent/src/agent/sender_allocation.rs | 51 ++++++++----------- 3 files changed, 51 insertions(+), 60 deletions(-) delete mode 100644 .sqlx/query-46dab6110aad21d2d87b4da3ea6f29bed709159e76d5d6ecd4370ac10d521cff.json create mode 100644 .sqlx/query-dbdcb666214a40762607e872c680bba5c3d01bc2106abe5839f1801d1683b8f6.json diff --git a/.sqlx/query-46dab6110aad21d2d87b4da3ea6f29bed709159e76d5d6ecd4370ac10d521cff.json b/.sqlx/query-46dab6110aad21d2d87b4da3ea6f29bed709159e76d5d6ecd4370ac10d521cff.json deleted file mode 100644 index fad6cb903..000000000 --- a/.sqlx/query-46dab6110aad21d2d87b4da3ea6f29bed709159e76d5d6ecd4370ac10d521cff.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n WITH rav AS (\n SELECT \n timestamp_ns \n FROM \n scalar_tap_ravs \n WHERE \n allocation_id = $1 \n AND sender_address = $2\n ) \n SELECT \n MAX(id), \n SUM(value) \n FROM \n scalar_tap_receipts \n WHERE \n allocation_id = $1 \n AND signer_address IN (SELECT unnest($3::text[]))\n AND CASE WHEN (\n SELECT \n timestamp_ns :: NUMERIC \n FROM \n rav\n ) IS NOT NULL THEN timestamp_ns > (\n SELECT \n timestamp_ns :: NUMERIC \n FROM \n rav\n ) ELSE TRUE END\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "max", - "type_info": "Int8" - }, - { - "ordinal": 1, - "name": "sum", - "type_info": "Numeric" - } - ], - "parameters": { - "Left": [ - "Bpchar", - "Bpchar", - "TextArray" - ] - }, - "nullable": [ - null, - null - ] - }, - "hash": "46dab6110aad21d2d87b4da3ea6f29bed709159e76d5d6ecd4370ac10d521cff" -} diff --git a/.sqlx/query-dbdcb666214a40762607e872c680bba5c3d01bc2106abe5839f1801d1683b8f6.json b/.sqlx/query-dbdcb666214a40762607e872c680bba5c3d01bc2106abe5839f1801d1683b8f6.json new file mode 100644 index 000000000..d9f97b776 --- /dev/null +++ b/.sqlx/query-dbdcb666214a40762607e872c680bba5c3d01bc2106abe5839f1801d1683b8f6.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH rav AS (\n SELECT\n timestamp_ns\n FROM\n scalar_tap_ravs\n WHERE\n allocation_id = $1\n AND sender_address = $2\n )\n SELECT\n MAX(id),\n SUM(value)\n FROM\n scalar_tap_receipts\n WHERE\n allocation_id = $1\n AND signer_address IN (SELECT unnest($3::text[]))\n AND CASE WHEN (\n SELECT\n timestamp_ns :: NUMERIC\n FROM\n rav\n ) IS NOT NULL THEN timestamp_ns > (\n SELECT\n timestamp_ns :: NUMERIC\n FROM\n rav\n ) ELSE TRUE END\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "max", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "sum", + "type_info": "Numeric" + } + ], + "parameters": { + "Left": [ + "Bpchar", + "Bpchar", + "TextArray" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "dbdcb666214a40762607e872c680bba5c3d01bc2106abe5839f1801d1683b8f6" +} diff --git a/tap-agent/src/agent/sender_allocation.rs b/tap-agent/src/agent/sender_allocation.rs index 07080305b..ab79e2d9e 100644 --- a/tap-agent/src/agent/sender_allocation.rs +++ b/tap-agent/src/agent/sender_allocation.rs @@ -271,31 +271,31 @@ impl SenderAllocationState { let res = sqlx::query!( r#" WITH rav AS ( - SELECT - timestamp_ns - FROM - scalar_tap_ravs - WHERE - allocation_id = $1 + SELECT + timestamp_ns + FROM + scalar_tap_ravs + WHERE + allocation_id = $1 AND sender_address = $2 - ) - SELECT - MAX(id), - SUM(value) - FROM - scalar_tap_receipts - WHERE - allocation_id = $1 + ) + SELECT + MAX(id), + SUM(value) + FROM + scalar_tap_receipts + WHERE + allocation_id = $1 AND signer_address IN (SELECT unnest($3::text[])) AND CASE WHEN ( - SELECT - timestamp_ns :: NUMERIC - FROM + SELECT + timestamp_ns :: NUMERIC + FROM rav ) IS NOT NULL THEN timestamp_ns > ( - SELECT - timestamp_ns :: NUMERIC - FROM + SELECT + timestamp_ns :: NUMERIC + FROM rav ) ELSE TRUE END "#, @@ -938,7 +938,7 @@ mod tests { .unwrap(); } - let (last_message_emitted, sender_account, _join_handle) = + let (_last_message_emitted, sender_account, _join_handle) = create_mock_sender_account().await; // create allocation @@ -963,15 +963,6 @@ mod tests { // check if the actor is actually stopped assert_eq!(sender_allocation.get_status(), ActorStatus::Stopped); - - // check if message is sent to sender account - assert_eq!( - last_message_emitted.lock().unwrap().last(), - Some(&SenderAccountMessage::UpdateReceiptFees( - *ALLOCATION_ID_0, - UnaggregatedReceipts::default() - )) - ); } #[sqlx::test(migrations = "../migrations")] From b8041633b5dc27ee5157b05ea6e0d534d44e003f Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Mon, 15 Apr 2024 14:58:56 -0300 Subject: [PATCH 41/41] test: add test case when the rav request fails Signed-off-by: Gustavo Inacio --- tap-agent/src/agent/sender_allocation.rs | 30 ++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tap-agent/src/agent/sender_allocation.rs b/tap-agent/src/agent/sender_allocation.rs index ab79e2d9e..c06b23a30 100644 --- a/tap-agent/src/agent/sender_allocation.rs +++ b/tap-agent/src/agent/sender_allocation.rs @@ -1090,4 +1090,34 @@ mod tests { // check if it fails assert!(result.is_ok()); } + + #[sqlx::test(migrations = "../migrations")] + async fn test_failed_rav_request(pgpool: PgPool) { + // Add receipts to the database. + for i in 0..10 { + let receipt = + create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, u64::max_value(), i.into()); + store_receipt(&pgpool, receipt.signed_receipt()) + .await + .unwrap(); + } + + // Create a sender_allocation. + let sender_allocation = + create_sender_allocation(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL, None).await; + + // Trigger a RAV request manually and wait for updated fees. + // this should fail because there's no receipt with valid timestamp + let total_unaggregated_fees = call!( + sender_allocation, + SenderAllocationMessage::TriggerRAVRequest + ) + .unwrap(); + + // expect the actor to keep running + assert_eq!(sender_allocation.get_status(), ActorStatus::Running); + + // Check that the unaggregated fees return the same value + assert_eq!(total_unaggregated_fees.value, 45u128); + } }