diff --git a/.sqlx/query-46dab6110aad21d2d87b4da3ea6f29bed709159e76d5d6ecd4370ac10d521cff.json b/.sqlx/query-46dab6110aad21d2d87b4da3ea6f29bed709159e76d5d6ecd4370ac10d521cff.json deleted file mode 100644 index fad6cb903..000000000 --- a/.sqlx/query-46dab6110aad21d2d87b4da3ea6f29bed709159e76d5d6ecd4370ac10d521cff.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n WITH rav AS (\n SELECT \n timestamp_ns \n FROM \n scalar_tap_ravs \n WHERE \n allocation_id = $1 \n AND sender_address = $2\n ) \n SELECT \n MAX(id), \n SUM(value) \n FROM \n scalar_tap_receipts \n WHERE \n allocation_id = $1 \n AND signer_address IN (SELECT unnest($3::text[]))\n AND CASE WHEN (\n SELECT \n timestamp_ns :: NUMERIC \n FROM \n rav\n ) IS NOT NULL THEN timestamp_ns > (\n SELECT \n timestamp_ns :: NUMERIC \n FROM \n rav\n ) ELSE TRUE END\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "max", - "type_info": "Int8" - }, - { - "ordinal": 1, - "name": "sum", - "type_info": "Numeric" - } - ], - "parameters": { - "Left": [ - "Bpchar", - "Bpchar", - "TextArray" - ] - }, - "nullable": [ - null, - null - ] - }, - "hash": "46dab6110aad21d2d87b4da3ea6f29bed709159e76d5d6ecd4370ac10d521cff" -} diff --git a/.sqlx/query-dbdcb666214a40762607e872c680bba5c3d01bc2106abe5839f1801d1683b8f6.json b/.sqlx/query-dbdcb666214a40762607e872c680bba5c3d01bc2106abe5839f1801d1683b8f6.json new file mode 100644 index 000000000..d9f97b776 --- /dev/null +++ b/.sqlx/query-dbdcb666214a40762607e872c680bba5c3d01bc2106abe5839f1801d1683b8f6.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH rav AS (\n SELECT\n timestamp_ns\n FROM\n scalar_tap_ravs\n WHERE\n allocation_id = $1\n AND sender_address = $2\n )\n SELECT\n MAX(id),\n SUM(value)\n FROM\n scalar_tap_receipts\n WHERE\n allocation_id = $1\n AND signer_address IN (SELECT unnest($3::text[]))\n AND CASE WHEN (\n SELECT\n timestamp_ns :: NUMERIC\n FROM\n rav\n ) IS NOT NULL THEN timestamp_ns > (\n SELECT\n timestamp_ns :: NUMERIC\n FROM\n rav\n ) ELSE TRUE END\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "max", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "sum", + "type_info": "Numeric" + } + ], + "parameters": { + "Left": [ + "Bpchar", + "Bpchar", + "TextArray" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "dbdcb666214a40762607e872c680bba5c3d01bc2106abe5839f1801d1683b8f6" +} diff --git a/Cargo.lock b/Cargo.lock index a5c20bed6..9ed6b2125 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2419,9 +2419,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" dependencies = [ "futures-channel", "futures-core", @@ -2434,9 +2434,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -2444,15 +2444,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" dependencies = [ "futures-core", "futures-task", @@ -2472,9 +2472,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-lite" @@ -2503,9 +2503,9 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", @@ -2514,15 +2514,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-timer" @@ -2536,9 +2536,9 @@ dependencies = [ [[package]] name = "futures-util" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-channel", "futures-core", @@ -3162,10 +3162,12 @@ dependencies = [ "ethers", "ethers-signers", "eventuals", + "futures", "graphql-http", "indexer-common", "jsonrpsee 0.20.2", "lazy_static", + "ractor", "reqwest", "serde", "serde_json", @@ -4843,6 +4845,21 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "ractor" +version = "0.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd876f0d609ba2ddc8a36136e9b81299312bd9fc9b71131381d16c9ce8e495a" +dependencies = [ + "async-trait", + "dashmap", + "futures", + "once_cell", + "rand 0.8.5", + "tokio", + "tracing", +] + [[package]] name = "radium" version = "0.7.0" @@ -6579,6 +6596,7 @@ dependencies = [ "signal-hook-registry", "socket2 0.5.4", "tokio-macros", + "tracing", "windows-sys", ] diff --git a/tap-agent/Cargo.toml b/tap-agent/Cargo.toml index 1200442ea..d946ef84a 100644 --- a/tap-agent/Cargo.toml +++ b/tap-agent/Cargo.toml @@ -50,8 +50,10 @@ tracing-subscriber = { version = "0.3", features = [ enum-as-inner = "0.6.0" ethers = "2.0.13" typetag = "0.2.14" +ractor = "0.9.7" [dev-dependencies] ethers-signers = "2.0.8" tempfile = "3.8.0" wiremock = "0.5.19" +futures = "0.3.30" diff --git a/tap-agent/src/agent.rs b/tap-agent/src/agent.rs index 345e56961..2e7dff338 100644 --- a/tap-agent/src/agent.rs +++ b/tap-agent/src/agent.rs @@ -3,90 +3,122 @@ use std::time::Duration; -use alloy_sol_types::eip712_domain; use indexer_common::prelude::{ escrow_accounts, indexer_allocations, DeploymentDetails, SubgraphClient, }; +use ractor::concurrency::JoinHandle; +use ractor::{Actor, ActorRef}; -use crate::{ - aggregator_endpoints, config, database, tap::sender_accounts_manager::SenderAccountsManager, +use crate::agent::sender_accounts_manager::{ + SenderAccountsManagerArgs, SenderAccountsManagerMessage, }; +use crate::config::{Cli, EscrowSubgraph, Ethereum, IndexerInfrastructure, NetworkSubgraph, Tap}; +use crate::{aggregator_endpoints, database, CONFIG, EIP_712_DOMAIN}; +use sender_accounts_manager::SenderAccountsManager; -pub async fn start_agent(config: &'static config::Cli) -> SenderAccountsManager { - let pgpool = database::connect(&config.postgres).await; +pub mod sender_account; +pub mod sender_accounts_manager; +pub mod sender_allocation; +pub mod sender_fee_tracker; +pub mod unaggregated_receipts; + +/// constant graph network used in subgraphs +const GRAPH_NETWORK_ID: u64 = 1; + +pub async fn start_agent() -> (ActorRef, JoinHandle<()>) { + let Cli { + ethereum: Ethereum { indexer_address }, + indexer_infrastructure: + IndexerInfrastructure { + graph_node_query_endpoint, + graph_node_status_endpoint, + .. + }, + postgres, + network_subgraph: + NetworkSubgraph { + network_subgraph_deployment, + network_subgraph_endpoint, + allocation_syncing_interval_ms, + }, + escrow_subgraph: + EscrowSubgraph { + escrow_subgraph_deployment, + escrow_subgraph_endpoint, + escrow_syncing_interval_ms, + }, + tap: Tap { + sender_aggregator_endpoints_file, + .. + }, + .. + } = &*CONFIG; + let pgpool = database::connect(postgres).await; let http_client = reqwest::Client::new(); let network_subgraph = Box::leak(Box::new(SubgraphClient::new( http_client.clone(), - config - .network_subgraph - .network_subgraph_deployment + network_subgraph_deployment .map(|deployment| { DeploymentDetails::for_graph_node( - &config.indexer_infrastructure.graph_node_status_endpoint, - &config.indexer_infrastructure.graph_node_query_endpoint, + graph_node_status_endpoint, + graph_node_query_endpoint, deployment, ) }) .transpose() .expect("Failed to parse graph node query endpoint and network subgraph deployment"), - DeploymentDetails::for_query_url(&config.network_subgraph.network_subgraph_endpoint) + DeploymentDetails::for_query_url(network_subgraph_endpoint) .expect("Failed to parse network subgraph endpoint"), ))); let indexer_allocations = indexer_allocations( network_subgraph, - config.ethereum.indexer_address, - 1, - Duration::from_millis(config.network_subgraph.allocation_syncing_interval_ms), + *indexer_address, + GRAPH_NETWORK_ID, + Duration::from_millis(*allocation_syncing_interval_ms), ); let escrow_subgraph = Box::leak(Box::new(SubgraphClient::new( http_client.clone(), - config - .escrow_subgraph - .escrow_subgraph_deployment + escrow_subgraph_deployment .map(|deployment| { DeploymentDetails::for_graph_node( - &config.indexer_infrastructure.graph_node_status_endpoint, - &config.indexer_infrastructure.graph_node_query_endpoint, + graph_node_status_endpoint, + graph_node_query_endpoint, deployment, ) }) .transpose() .expect("Failed to parse graph node query endpoint and escrow subgraph deployment"), - DeploymentDetails::for_query_url(&config.escrow_subgraph.escrow_subgraph_endpoint) + DeploymentDetails::for_query_url(escrow_subgraph_endpoint) .expect("Failed to parse escrow subgraph endpoint"), ))); let escrow_accounts = escrow_accounts( escrow_subgraph, - config.ethereum.indexer_address, - Duration::from_millis(config.escrow_subgraph.escrow_syncing_interval_ms), + *indexer_address, + Duration::from_millis(*escrow_syncing_interval_ms), false, ); // TODO: replace with a proper implementation once the gateway registry contract is ready - let sender_aggregator_endpoints = aggregator_endpoints::load_aggregator_endpoints( - config.tap.sender_aggregator_endpoints_file.clone(), - ); + let sender_aggregator_endpoints = + aggregator_endpoints::load_aggregator_endpoints(sender_aggregator_endpoints_file.clone()); - let tap_eip712_domain_separator = eip712_domain! { - name: "TAP", - version: "1", - chain_id: config.receipts.receipts_verifier_chain_id, - verifying_contract: config.receipts.receipts_verifier_address, - }; - - SenderAccountsManager::new( - config, + let args = SenderAccountsManagerArgs { + config: &CONFIG, + domain_separator: EIP_712_DOMAIN.clone(), pgpool, indexer_allocations, escrow_accounts, escrow_subgraph, - tap_eip712_domain_separator, sender_aggregator_endpoints, - ) - .await + prefix: None, + }; + + SenderAccountsManager::spawn(None, SenderAccountsManager, args) + .await + .expect("Failed to start sender accounts manager actor.") } diff --git a/tap-agent/src/agent/sender_account.rs b/tap-agent/src/agent/sender_account.rs new file mode 100644 index 000000000..3cf0fcfb6 --- /dev/null +++ b/tap-agent/src/agent/sender_account.rs @@ -0,0 +1,630 @@ +// Copyright 2023-, GraphOps and Semiotic Labs. +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::HashSet; + +use alloy_sol_types::Eip712Domain; +use anyhow::Result; +use eventuals::{Eventual, EventualExt, PipeHandle}; +use indexer_common::{escrow_accounts::EscrowAccounts, prelude::SubgraphClient}; +use ractor::{call, Actor, ActorProcessingErr, ActorRef, SupervisionEvent}; +use sqlx::PgPool; +use thegraph::types::Address; +use tracing::{error, Level}; + +use super::sender_allocation::{SenderAllocation, SenderAllocationArgs}; +use crate::agent::sender_allocation::SenderAllocationMessage; +use crate::agent::sender_fee_tracker::SenderFeeTracker; +use crate::agent::unaggregated_receipts::UnaggregatedReceipts; +use crate::{ + config::{self}, + tap::escrow_adapter::EscrowAdapter, +}; + +#[derive(Debug)] +pub enum SenderAccountMessage { + UpdateAllocationIds(HashSet
), + UpdateReceiptFees(Address, UnaggregatedReceipts), + #[cfg(test)] + GetSenderFeeTracker(ractor::RpcReplyPort), +} + +/// A SenderAccount manages the receipts accounting between the indexer and the sender across +/// multiple allocations. +/// +/// Manages the lifecycle of Scalar TAP for the SenderAccount, including: +/// - Monitoring new receipts and keeping track of the cumulative unaggregated fees across +/// allocations. +/// - Requesting RAVs from the sender's TAP aggregator once the cumulative unaggregated fees reach a +/// certain threshold. +/// - Requesting the last RAV from the sender's TAP aggregator for all EOL allocations. +pub struct SenderAccount; + +pub struct SenderAccountArgs { + pub config: &'static config::Cli, + pub pgpool: PgPool, + pub sender_id: Address, + pub escrow_accounts: Eventual, + pub indexer_allocations: Eventual>, + pub escrow_subgraph: &'static SubgraphClient, + pub domain_separator: Eip712Domain, + pub sender_aggregator_endpoint: String, + pub allocation_ids: HashSet
, + pub prefix: Option, +} +pub struct State { + prefix: Option, + sender_fee_tracker: SenderFeeTracker, + allocation_ids: HashSet
, + _indexer_allocations_handle: PipeHandle, + sender: Address, + + //Eventuals + escrow_accounts: Eventual, + + escrow_subgraph: &'static SubgraphClient, + escrow_adapter: EscrowAdapter, + domain_separator: Eip712Domain, + config: &'static config::Cli, + pgpool: PgPool, + sender_aggregator_endpoint: String, +} + +impl State { + async fn create_sender_allocation( + &self, + sender_account_ref: ActorRef, + allocation_id: Address, + ) -> Result<()> { + tracing::trace!( + %self.sender, + %allocation_id, + "SenderAccount is creating allocation." + ); + let args = SenderAllocationArgs { + config: self.config, + pgpool: self.pgpool.clone(), + allocation_id, + sender: self.sender, + escrow_accounts: self.escrow_accounts.clone(), + escrow_subgraph: self.escrow_subgraph, + escrow_adapter: self.escrow_adapter.clone(), + domain_separator: self.domain_separator.clone(), + sender_aggregator_endpoint: self.sender_aggregator_endpoint.clone(), + sender_account_ref: sender_account_ref.clone(), + }; + + SenderAllocation::spawn_linked( + Some(self.format_sender_allocation(&allocation_id)), + SenderAllocation, + args, + sender_account_ref.get_cell(), + ) + .await?; + Ok(()) + } + fn format_sender_allocation(&self, allocation_id: &Address) -> String { + let mut sender_allocation_id = String::new(); + if let Some(prefix) = &self.prefix { + sender_allocation_id.push_str(prefix); + sender_allocation_id.push(':'); + } + sender_allocation_id.push_str(&format!("{}:{}", self.sender, allocation_id)); + sender_allocation_id + } + + async fn rav_requester_single(&mut self) -> Result<()> { + let Some(allocation_id) = self.sender_fee_tracker.get_heaviest_allocation_id() else { + anyhow::bail!("Error while getting the heaviest allocation because none has unaggregated fees tracked"); + }; + let sender_allocation_id = self.format_sender_allocation(&allocation_id); + let allocation = ActorRef::::where_is(sender_allocation_id); + + let Some(allocation) = allocation else { + anyhow::bail!("Error while getting allocation actor with most unaggregated fees"); + }; + // we call and wait for the response so we don't process anymore update + let result = call!(allocation, SenderAllocationMessage::TriggerRAVRequest)?; + + self.sender_fee_tracker.update(allocation_id, result.value); + Ok(()) + } +} + +#[async_trait::async_trait] +impl Actor for SenderAccount { + type Msg = SenderAccountMessage; + type State = State; + type Arguments = SenderAccountArgs; + + async fn pre_start( + &self, + myself: ActorRef, + SenderAccountArgs { + config, + pgpool, + sender_id, + escrow_accounts, + indexer_allocations, + escrow_subgraph, + domain_separator, + sender_aggregator_endpoint, + allocation_ids, + prefix, + }: Self::Arguments, + ) -> std::result::Result { + let clone = myself.clone(); + let _indexer_allocations_handle = + indexer_allocations + .clone() + .pipe_async(move |allocation_ids| { + let myself = clone.clone(); + async move { + // Update the allocation_ids + myself + .cast(SenderAccountMessage::UpdateAllocationIds(allocation_ids)) + .unwrap_or_else(|e| { + error!("Error while updating allocation_ids: {:?}", e); + }); + } + }); + + let escrow_adapter = EscrowAdapter::new(escrow_accounts.clone(), sender_id); + + let state = State { + sender_fee_tracker: SenderFeeTracker::default(), + allocation_ids: allocation_ids.clone(), + _indexer_allocations_handle, + prefix, + escrow_accounts, + escrow_subgraph, + escrow_adapter, + domain_separator, + sender_aggregator_endpoint, + config, + pgpool, + sender: sender_id, + }; + + for allocation_id in &allocation_ids { + // Create a sender allocation for each allocation + state + .create_sender_allocation(myself.clone(), *allocation_id) + .await?; + } + + tracing::info!(sender = %sender_id, "SenderAccount created!"); + Ok(state) + } + + async fn handle( + &self, + myself: ActorRef, + message: Self::Msg, + state: &mut Self::State, + ) -> std::result::Result<(), ActorProcessingErr> { + tracing::span!( + Level::TRACE, + "SenderAccount handle()", + sender = %state.sender, + ); + tracing::trace!( + message = ?message, + "New SenderAccount message" + ); + match message { + SenderAccountMessage::UpdateReceiptFees(allocation_id, unaggregated_fees) => { + let tracker = &mut state.sender_fee_tracker; + tracker.update(allocation_id, unaggregated_fees.value); + + if tracker.get_total_fee() >= state.config.tap.rav_request_trigger_value.into() { + tracing::debug!( + total_fee = tracker.get_total_fee(), + trigger_value = state.config.tap.rav_request_trigger_value, + "Total fee greater than the trigger value. Triggering RAV request" + ); + state.rav_requester_single().await?; + } + } + SenderAccountMessage::UpdateAllocationIds(allocation_ids) => { + // Create new sender allocations + for allocation_id in allocation_ids.difference(&state.allocation_ids) { + state + .create_sender_allocation(myself.clone(), *allocation_id) + .await?; + } + + // Remove sender allocations + for allocation_id in state.allocation_ids.difference(&allocation_ids) { + if let Some(sender_handle) = ActorRef::::where_is( + state.format_sender_allocation(allocation_id), + ) { + tracing::trace!(%allocation_id, "SenderAccount shutting down SenderAllocation"); + sender_handle.stop(None); + } + } + + tracing::trace!( + old_ids= ?state.allocation_ids, + new_ids = ?allocation_ids, + "Updating allocation ids" + ); + state.allocation_ids = allocation_ids; + } + #[cfg(test)] + SenderAccountMessage::GetSenderFeeTracker(reply) => { + if !reply.is_closed() { + let _ = reply.send(state.sender_fee_tracker.clone()); + } + } + } + Ok(()) + } + + // we define the supervisor event to overwrite the default behavior which + // is shutdown the supervisor on actor termination events + async fn handle_supervisor_evt( + &self, + myself: ActorRef, + message: SupervisionEvent, + state: &mut Self::State, + ) -> std::result::Result<(), ActorProcessingErr> { + tracing::trace!( + sender = %state.sender, + message = ?message, + "New SenderAccount supervision event" + ); + + match message { + SupervisionEvent::ActorTerminated(cell, _, _) => { + // what to do in case of termination or panic? + let sender_allocation = cell.get_name(); + tracing::warn!(?sender_allocation, "Actor SenderAllocation was terminated"); + + let Some(allocation_id) = cell.get_name() else { + tracing::error!("SenderAllocation doesn't have a name"); + return Ok(()); + }; + let Some(allocation_id) = allocation_id.split(':').last() else { + tracing::error!(%allocation_id, "Could not extract allocation_id from name"); + return Ok(()); + }; + let Ok(allocation_id) = Address::parse_checksummed(allocation_id, None) else { + tracing::error!(%allocation_id, "Could not convert allocation_id to Address"); + return Ok(()); + }; + + let tracker = &mut state.sender_fee_tracker; + tracker.update(allocation_id, 0); + } + SupervisionEvent::ActorPanicked(cell, error) => { + let sender_allocation = cell.get_name(); + tracing::warn!( + ?sender_allocation, + ?error, + "Actor SenderAllocation panicked. Restarting..." + ); + let Some(allocation_id) = cell.get_name() else { + tracing::error!("SenderAllocation doesn't have a name"); + return Ok(()); + }; + let Some(allocation_id) = allocation_id.split(':').last() else { + tracing::error!(%allocation_id, "Could not extract allocation_id from name"); + return Ok(()); + }; + let Ok(allocation_id) = Address::parse_checksummed(allocation_id, None) else { + tracing::error!(%allocation_id, "Could not convert allocation_id to Address"); + return Ok(()); + }; + + state + .create_sender_allocation(myself.clone(), allocation_id) + .await?; + } + _ => {} + } + Ok(()) + } +} + +#[cfg(test)] +pub mod tests { + use super::{SenderAccount, SenderAccountArgs, SenderAccountMessage}; + use crate::agent::sender_accounts_manager::NewReceiptNotification; + use crate::agent::sender_allocation::SenderAllocationMessage; + use crate::agent::unaggregated_receipts::UnaggregatedReceipts; + use crate::config; + use crate::tap::test_utils::{ + ALLOCATION_ID_0, INDEXER, SENDER, SIGNER, TAP_EIP712_DOMAIN_SEPARATOR, + }; + use alloy_primitives::Address; + use eventuals::Eventual; + use indexer_common::escrow_accounts::EscrowAccounts; + use indexer_common::prelude::{DeploymentDetails, SubgraphClient}; + use ractor::concurrency::JoinHandle; + use ractor::{Actor, ActorProcessingErr, ActorRef, ActorStatus}; + use sqlx::PgPool; + use std::collections::{HashMap, HashSet}; + use std::sync::atomic::{AtomicBool, AtomicU32}; + use std::sync::{Arc, Mutex}; + use std::time::Duration; + + // we implement the PartialEq and Eq traits for SenderAccountMessage to be able to compare + impl Eq for SenderAccountMessage {} + + impl PartialEq for SenderAccountMessage { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Self::UpdateAllocationIds(l0), Self::UpdateAllocationIds(r0)) => l0 == r0, + (Self::UpdateReceiptFees(l0, l1), Self::UpdateReceiptFees(r0, r1)) => { + l0 == r0 && l1 == r1 + } + _ => core::mem::discriminant(self) == core::mem::discriminant(other), + } + } + } + + pub static PREFIX_ID: AtomicU32 = AtomicU32::new(0); + const DUMMY_URL: &str = "http://localhost:1234"; + const TRIGGER_VALUE: u128 = 500; + + async fn create_sender_account( + pgpool: PgPool, + initial_allocation: HashSet
, + ) -> ( + ActorRef, + tokio::task::JoinHandle<()>, + String, + ) { + let config = Box::leak(Box::new(config::Cli { + config: None, + ethereum: config::Ethereum { + indexer_address: INDEXER.1, + }, + tap: config::Tap { + rav_request_trigger_value: TRIGGER_VALUE as u64, + rav_request_timestamp_buffer_ms: 1, + rav_request_timeout_secs: 5, + ..Default::default() + }, + ..Default::default() + })); + + let escrow_subgraph = Box::leak(Box::new(SubgraphClient::new( + reqwest::Client::new(), + None, + DeploymentDetails::for_query_url(DUMMY_URL).unwrap(), + ))); + + let escrow_accounts_eventual = Eventual::from_value(EscrowAccounts::new( + HashMap::from([(SENDER.1, 1000.into())]), + HashMap::from([(SENDER.1, vec![SIGNER.1])]), + )); + + let prefix = format!( + "test-{}", + PREFIX_ID.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + ); + + let args = SenderAccountArgs { + config, + pgpool, + sender_id: SENDER.1, + escrow_accounts: escrow_accounts_eventual, + indexer_allocations: Eventual::from_value(initial_allocation), + escrow_subgraph, + domain_separator: TAP_EIP712_DOMAIN_SEPARATOR.clone(), + sender_aggregator_endpoint: DUMMY_URL.to_string(), + allocation_ids: HashSet::new(), + prefix: Some(prefix.clone()), + }; + + let (sender, handle) = SenderAccount::spawn(Some(prefix.clone()), SenderAccount, args) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(10)).await; + (sender, handle, prefix) + } + + #[sqlx::test(migrations = "../migrations")] + async fn test_update_allocation_ids(pgpool: PgPool) { + let (sender_account, handle, prefix) = create_sender_account(pgpool, HashSet::new()).await; + + // we expect it to create a sender allocation + sender_account + .cast(SenderAccountMessage::UpdateAllocationIds( + vec![*ALLOCATION_ID_0].into_iter().collect(), + )) + .unwrap(); + + tokio::time::sleep(Duration::from_millis(10)).await; + + // verify if create sender account + let sender_allocation_id = format!("{}:{}:{}", prefix.clone(), SENDER.1, *ALLOCATION_ID_0); + let actor_ref = ActorRef::::where_is(sender_allocation_id.clone()); + assert!(actor_ref.is_some()); + + sender_account + .cast(SenderAccountMessage::UpdateAllocationIds(HashSet::new())) + .unwrap(); + + tokio::time::sleep(Duration::from_millis(100)).await; + + let actor_ref = ActorRef::::where_is(sender_allocation_id.clone()); + assert!(actor_ref.is_none()); + + // safely stop the manager + sender_account.stop_and_wait(None, None).await.unwrap(); + + handle.await.unwrap(); + } + + pub struct MockSenderAllocation { + triggered_rav_request: Arc, + receipts: Arc>>, + } + + impl MockSenderAllocation { + pub fn new_with_triggered_rav_request() -> (Self, Arc) { + let triggered_rav_request = Arc::new(AtomicBool::new(false)); + ( + Self { + triggered_rav_request: triggered_rav_request.clone(), + receipts: Arc::new(Mutex::new(Vec::new())), + }, + triggered_rav_request, + ) + } + + pub fn new_with_receipts() -> (Self, Arc>>) { + let receipts = Arc::new(Mutex::new(Vec::new())); + ( + Self { + triggered_rav_request: Arc::new(AtomicBool::new(false)), + receipts: receipts.clone(), + }, + receipts, + ) + } + } + + #[async_trait::async_trait] + impl Actor for MockSenderAllocation { + type Msg = SenderAllocationMessage; + type State = (); + type Arguments = (); + + async fn pre_start( + &self, + _myself: ActorRef, + _allocation_ids: Self::Arguments, + ) -> Result { + Ok(()) + } + + async fn handle( + &self, + _myself: ActorRef, + message: Self::Msg, + _state: &mut Self::State, + ) -> Result<(), ActorProcessingErr> { + match message { + SenderAllocationMessage::TriggerRAVRequest(reply) => { + self.triggered_rav_request + .store(true, std::sync::atomic::Ordering::SeqCst); + reply.send(UnaggregatedReceipts::default())?; + } + SenderAllocationMessage::NewReceipt(receipt) => { + self.receipts.lock().unwrap().push(receipt); + } + _ => {} + } + Ok(()) + } + } + + async fn create_mock_sender_allocation( + prefix: String, + sender: Address, + allocation: Address, + ) -> ( + Arc, + ActorRef, + JoinHandle<()>, + ) { + let (mock_sender_allocation, triggered_rav_request) = + MockSenderAllocation::new_with_triggered_rav_request(); + + let name = format!("{}:{}:{}", prefix, sender, allocation); + let (sender_account, join_handle) = + MockSenderAllocation::spawn(Some(name), mock_sender_allocation, ()) + .await + .unwrap(); + (triggered_rav_request, sender_account, join_handle) + } + + #[sqlx::test(migrations = "../migrations")] + async fn test_update_receipt_fees_no_rav(pgpool: PgPool) { + let (sender_account, handle, prefix) = create_sender_account(pgpool, HashSet::new()).await; + + let (triggered_rav_request, allocation, allocation_handle) = + create_mock_sender_allocation(prefix, SENDER.1, *ALLOCATION_ID_0).await; + + // create a fake sender allocation + sender_account + .cast(SenderAccountMessage::UpdateReceiptFees( + *ALLOCATION_ID_0, + UnaggregatedReceipts { + value: TRIGGER_VALUE - 1, + last_id: 10, + }, + )) + .unwrap(); + + tokio::time::sleep(Duration::from_millis(10)).await; + + assert!(!triggered_rav_request.load(std::sync::atomic::Ordering::SeqCst)); + + allocation.stop_and_wait(None, None).await.unwrap(); + allocation_handle.await.unwrap(); + + sender_account.stop_and_wait(None, None).await.unwrap(); + handle.await.unwrap(); + } + + #[sqlx::test(migrations = "../migrations")] + async fn test_update_receipt_fees_trigger_rav(pgpool: PgPool) { + let (sender_account, handle, prefix) = create_sender_account(pgpool, HashSet::new()).await; + + let (triggered_rav_request, allocation, allocation_handle) = + create_mock_sender_allocation(prefix, SENDER.1, *ALLOCATION_ID_0).await; + + // create a fake sender allocation + sender_account + .cast(SenderAccountMessage::UpdateReceiptFees( + *ALLOCATION_ID_0, + UnaggregatedReceipts { + value: TRIGGER_VALUE, + last_id: 10, + }, + )) + .unwrap(); + + tokio::time::sleep(Duration::from_millis(10)).await; + + assert!(triggered_rav_request.load(std::sync::atomic::Ordering::SeqCst)); + + allocation.stop_and_wait(None, None).await.unwrap(); + allocation_handle.await.unwrap(); + + sender_account.stop_and_wait(None, None).await.unwrap(); + handle.await.unwrap(); + } + + #[sqlx::test(migrations = "../migrations")] + async fn test_remove_sender_account(pgpool: PgPool) { + let (sender_account, handle, prefix) = + create_sender_account(pgpool, vec![*ALLOCATION_ID_0].into_iter().collect()).await; + + // check if allocation exists + let sender_allocation_id = format!("{}:{}:{}", prefix.clone(), SENDER.1, *ALLOCATION_ID_0); + let Some(sender_allocation) = + ActorRef::::where_is(sender_allocation_id.clone()) + else { + panic!("Sender allocation was not created"); + }; + + // stop + sender_account.stop_and_wait(None, None).await.unwrap(); + + // check if sender_account is stopped + assert_eq!(sender_account.get_status(), ActorStatus::Stopped); + + tokio::time::sleep(Duration::from_millis(10)).await; + + // check if sender_allocation is also stopped + assert_eq!(sender_allocation.get_status(), ActorStatus::Stopped); + + handle.await.unwrap(); + } +} diff --git a/tap-agent/src/agent/sender_accounts_manager.rs b/tap-agent/src/agent/sender_accounts_manager.rs new file mode 100644 index 000000000..a4d75b0bb --- /dev/null +++ b/tap-agent/src/agent/sender_accounts_manager.rs @@ -0,0 +1,764 @@ +// Copyright 2023-, GraphOps and Semiotic Labs. +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::HashSet; +use std::{collections::HashMap, str::FromStr}; + +use crate::agent::sender_allocation::SenderAllocationMessage; +use alloy_sol_types::Eip712Domain; +use anyhow::anyhow; +use anyhow::Result; +use eventuals::{Eventual, EventualExt, PipeHandle}; +use indexer_common::escrow_accounts::EscrowAccounts; +use indexer_common::prelude::{Allocation, SubgraphClient}; +use ractor::{Actor, ActorCell, ActorProcessingErr, ActorRef, SupervisionEvent}; +use serde::Deserialize; +use sqlx::{postgres::PgListener, PgPool}; +use thegraph::types::Address; +use tokio::select; +use tracing::{error, warn}; + +use super::sender_account::{SenderAccount, SenderAccountArgs, SenderAccountMessage}; +use crate::config; + +#[derive(Deserialize, Debug)] +pub struct NewReceiptNotification { + pub id: u64, + pub allocation_id: Address, + pub signer_address: Address, + pub timestamp_ns: u64, + pub value: u128, +} + +pub struct SenderAccountsManager; + +#[derive(Debug)] +pub enum SenderAccountsManagerMessage { + UpdateSenderAccounts(HashSet
), +} + +pub struct SenderAccountsManagerArgs { + pub config: &'static config::Cli, + pub domain_separator: Eip712Domain, + + pub pgpool: PgPool, + pub indexer_allocations: Eventual>, + pub escrow_accounts: Eventual, + pub escrow_subgraph: &'static SubgraphClient, + pub sender_aggregator_endpoints: HashMap, + + pub prefix: Option, +} + +pub struct State { + sender_ids: HashSet
, + new_receipts_watcher_handle: tokio::task::JoinHandle<()>, + _eligible_allocations_senders_pipe: PipeHandle, + + config: &'static config::Cli, + domain_separator: Eip712Domain, + pgpool: PgPool, + indexer_allocations: Eventual>, + escrow_accounts: Eventual, + escrow_subgraph: &'static SubgraphClient, + sender_aggregator_endpoints: HashMap, + prefix: Option, +} + +#[async_trait::async_trait] +impl Actor for SenderAccountsManager { + type Msg = SenderAccountsManagerMessage; + type State = State; + type Arguments = SenderAccountsManagerArgs; + + async fn pre_start( + &self, + myself: ActorRef, + SenderAccountsManagerArgs { + config, + domain_separator, + indexer_allocations, + pgpool, + escrow_accounts, + escrow_subgraph, + sender_aggregator_endpoints, + prefix, + }: Self::Arguments, + ) -> std::result::Result { + let indexer_allocations = indexer_allocations.map(|allocations| async move { + allocations.keys().cloned().collect::>() + }); + let mut pglistener = PgListener::connect_with(&pgpool.clone()).await.unwrap(); + pglistener + .listen("scalar_tap_receipt_notification") + .await + .expect( + "should be able to subscribe to Postgres Notify events on the channel \ + 'scalar_tap_receipt_notification'", + ); + // Start the new_receipts_watcher task that will consume from the `pglistener` + let new_receipts_watcher_handle = tokio::spawn(new_receipts_watcher( + pglistener, + escrow_accounts.clone(), + prefix.clone(), + )); + let clone = myself.clone(); + let _eligible_allocations_senders_pipe = + escrow_accounts.clone().pipe_async(move |escrow_accounts| { + let myself = clone.clone(); + + async move { + myself + .cast(SenderAccountsManagerMessage::UpdateSenderAccounts( + escrow_accounts.get_senders(), + )) + .unwrap_or_else(|e| { + error!("Error while updating sender_accounts: {:?}", e); + }); + } + }); + + let mut state = State { + config, + domain_separator, + sender_ids: HashSet::new(), + new_receipts_watcher_handle, + _eligible_allocations_senders_pipe, + pgpool, + indexer_allocations, + escrow_accounts, + escrow_subgraph, + sender_aggregator_endpoints, + prefix, + }; + let sender_allocation = select! { + sender_allocation = state.get_pending_sender_allocation_id() => sender_allocation, + _ = tokio::time::sleep(std::time::Duration::from_secs(30)) => { + panic!("Timeout while getting pending sender allocation ids"); + } + }; + + for (sender_id, allocation_ids) in sender_allocation { + state.sender_ids.insert(sender_id); + state + .create_sender_account(myself.get_cell(), sender_id, allocation_ids) + .await?; + } + + tracing::info!("SenderAccountManager created!"); + Ok(state) + } + async fn post_stop( + &self, + _: ActorRef, + state: &mut Self::State, + ) -> std::result::Result<(), ActorProcessingErr> { + // Abort the notification watcher on drop. Otherwise it may panic because the PgPool could + // get dropped before. (Observed in tests) + state.new_receipts_watcher_handle.abort(); + Ok(()) + } + + async fn handle( + &self, + myself: ActorRef, + msg: Self::Msg, + state: &mut Self::State, + ) -> std::result::Result<(), ActorProcessingErr> { + tracing::trace!( + message = ?msg, + "New SenderAccountManager message" + ); + + match msg { + SenderAccountsManagerMessage::UpdateSenderAccounts(target_senders) => { + // Create new sender accounts + for sender in target_senders.difference(&state.sender_ids) { + state + .create_sender_account(myself.get_cell(), *sender, HashSet::new()) + .await?; + } + + // Remove sender accounts + for sender in state.sender_ids.difference(&target_senders) { + if let Some(sender_handle) = ActorRef::::where_is( + state.format_sender_account(sender), + ) { + sender_handle.stop(None); + } + } + + state.sender_ids = target_senders; + } + } + Ok(()) + } + + // we define the supervisor event to overwrite the default behavior which + // is shutdown the supervisor on actor termination events + async fn handle_supervisor_evt( + &self, + myself: ActorRef, + message: SupervisionEvent, + state: &mut Self::State, + ) -> std::result::Result<(), ActorProcessingErr> { + match message { + SupervisionEvent::ActorTerminated(cell, _, reason) => { + let sender_id = cell.get_name(); + tracing::info!(?sender_id, ?reason, "Actor SenderAccount was terminated") + } + SupervisionEvent::ActorPanicked(cell, error) => { + let sender_id = cell.get_name(); + tracing::warn!( + ?sender_id, + ?error, + "Actor SenderAccount panicked. Restarting..." + ); + let Some(sender_id) = cell.get_name() else { + tracing::error!("SenderAllocation doesn't have a name"); + return Ok(()); + }; + let Some(sender_id) = sender_id.split(':').last() else { + tracing::error!(%sender_id, "Could not extract sender_id from name"); + return Ok(()); + }; + let Ok(sender_id) = Address::parse_checksummed(sender_id, None) else { + tracing::error!(%sender_id, "Could not convert sender_id to Address"); + return Ok(()); + }; + + let mut sender_allocation = select! { + sender_allocation = state.get_pending_sender_allocation_id() => sender_allocation, + _ = tokio::time::sleep(std::time::Duration::from_secs(30)) => { + tracing::error!("Timeout while getting pending sender allocation ids"); + return Ok(()); + } + }; + + let allocations = sender_allocation + .remove(&sender_id) + .unwrap_or(HashSet::new()); + + state + .create_sender_account(myself.get_cell(), sender_id, allocations) + .await?; + } + _ => {} + } + Ok(()) + } +} + +impl State { + fn format_sender_account(&self, sender: &Address) -> String { + let mut sender_allocation_id = String::new(); + if let Some(prefix) = &self.prefix { + sender_allocation_id.push_str(prefix); + sender_allocation_id.push(':'); + } + sender_allocation_id.push_str(&format!("{}", sender)); + sender_allocation_id + } + + async fn create_sender_account( + &self, + supervisor: ActorCell, + sender_id: Address, + allocation_ids: HashSet
, + ) -> anyhow::Result<()> { + let args = self.new_sender_account_args(&sender_id, allocation_ids)?; + SenderAccount::spawn_linked( + Some(self.format_sender_account(&sender_id)), + SenderAccount, + args, + supervisor, + ) + .await?; + Ok(()) + } + + async fn get_pending_sender_allocation_id(&self) -> HashMap> { + let escrow_accounts_snapshot = self + .escrow_accounts + .value() + .await + .expect("Should get escrow accounts from Eventual"); + + // Gather all outstanding receipts and unfinalized RAVs from the database. + // Used to create SenderAccount instances for all senders that have unfinalized allocations + // and try to finalize them if they have become ineligible. + + // First we accumulate all allocations for each sender. This is because we may have more + // than one signer per sender in DB. + let mut unfinalized_sender_allocations_map: HashMap> = + HashMap::new(); + + let receipts_signer_allocations_in_db = sqlx::query!( + r#" + SELECT DISTINCT + signer_address, + ( + SELECT ARRAY + ( + SELECT DISTINCT allocation_id + FROM scalar_tap_receipts + WHERE signer_address = top.signer_address + ) + ) AS allocation_ids + FROM scalar_tap_receipts AS top + "# + ) + .fetch_all(&self.pgpool) + .await + .expect("should be able to fetch pending receipts from the database"); + + for row in receipts_signer_allocations_in_db { + let allocation_ids = row + .allocation_ids + .expect("all receipts should have an allocation_id") + .iter() + .map(|allocation_id| { + Address::from_str(allocation_id) + .expect("allocation_id should be a valid address") + }) + .collect::>(); + let signer_id = Address::from_str(&row.signer_address) + .expect("signer_address should be a valid address"); + let sender_id = escrow_accounts_snapshot + .get_sender_for_signer(&signer_id) + .expect("should be able to get sender from signer"); + + // Accumulate allocations for the sender + unfinalized_sender_allocations_map + .entry(sender_id) + .or_default() + .extend(allocation_ids); + } + + let nonfinal_ravs_sender_allocations_in_db = sqlx::query!( + r#" + SELECT DISTINCT + sender_address, + ( + SELECT ARRAY + ( + SELECT DISTINCT allocation_id + FROM scalar_tap_ravs + WHERE sender_address = top.sender_address + ) + ) AS allocation_id + FROM scalar_tap_ravs AS top + "# + ) + .fetch_all(&self.pgpool) + .await + .expect("should be able to fetch unfinalized RAVs from the database"); + + for row in nonfinal_ravs_sender_allocations_in_db { + let allocation_ids = row + .allocation_id + .expect("all RAVs should have an allocation_id") + .iter() + .map(|allocation_id| { + Address::from_str(allocation_id) + .expect("allocation_id should be a valid address") + }) + .collect::>(); + let sender_id = Address::from_str(&row.sender_address) + .expect("sender_address should be a valid address"); + + // Accumulate allocations for the sender + unfinalized_sender_allocations_map + .entry(sender_id) + .or_default() + .extend(allocation_ids); + } + unfinalized_sender_allocations_map + } + fn new_sender_account_args( + &self, + sender_id: &Address, + allocation_ids: HashSet
, + ) -> Result { + Ok(SenderAccountArgs { + config: self.config, + pgpool: self.pgpool.clone(), + sender_id: *sender_id, + escrow_accounts: self.escrow_accounts.clone(), + indexer_allocations: self.indexer_allocations.clone(), + escrow_subgraph: self.escrow_subgraph, + domain_separator: self.domain_separator.clone(), + sender_aggregator_endpoint: self + .sender_aggregator_endpoints + .get(sender_id) + .ok_or_else(|| { + anyhow!( + "No sender_aggregator_endpoint found for sender {}", + sender_id + ) + })? + .clone(), + allocation_ids, + prefix: self.prefix.clone(), + }) + } +} + +/// Continuously listens for new receipt notifications from Postgres and forwards them to the +/// corresponding SenderAccount. +async fn new_receipts_watcher( + mut pglistener: PgListener, + escrow_accounts: Eventual, + prefix: Option, +) { + loop { + // TODO: recover from errors or shutdown the whole program? + let pg_notification = pglistener.recv().await.expect( + "should be able to receive Postgres Notify events on the channel \ + 'scalar_tap_receipt_notification'", + ); + let new_receipt_notification: NewReceiptNotification = + serde_json::from_str(pg_notification.payload()).expect( + "should be able to deserialize the Postgres Notify event payload as a \ + NewReceiptNotification", + ); + + tracing::debug!( + notification = ?new_receipt_notification, + "New receipt notification detected!" + ); + + let Ok(sender_address) = escrow_accounts + .value() + .await + .expect("should be able to get escrow accounts") + .get_sender_for_signer(&new_receipt_notification.signer_address) + else { + error!( + "No sender address found for receipt signer address {}. \ + This should not happen.", + new_receipt_notification.signer_address + ); + // TODO: save the receipt in the failed receipts table? + continue; + }; + + let allocation_id = &new_receipt_notification.allocation_id; + + let actor_name = format!( + "{}{sender_address}:{allocation_id}", + prefix + .as_ref() + .map_or(String::default(), |prefix| format!("{prefix}:")) + ); + + if let Some(sender_allocation) = ActorRef::::where_is(actor_name) { + if let Err(e) = sender_allocation.cast(SenderAllocationMessage::NewReceipt( + new_receipt_notification, + )) { + error!( + "Error while forwarding new receipt notification to sender_allocation: {:?}", + e + ); + } + } else { + warn!( + "No sender_allocation found for sender_address {}, allocation_id {} to process new \ + receipt notification. This should not happen.", + sender_address, + allocation_id + ); + } + } +} + +#[cfg(test)] +mod tests { + use super::{ + new_receipts_watcher, SenderAccountsManager, SenderAccountsManagerArgs, + SenderAccountsManagerMessage, State, + }; + use crate::agent::sender_account::tests::{MockSenderAllocation, PREFIX_ID}; + use crate::agent::sender_account::SenderAccountMessage; + use crate::config; + use crate::tap::test_utils::{ + create_rav, create_received_receipt, store_rav, store_receipt, ALLOCATION_ID_0, + ALLOCATION_ID_1, INDEXER, SENDER, SENDER_2, SIGNER, TAP_EIP712_DOMAIN_SEPARATOR, + }; + use alloy_primitives::Address; + use eventuals::{Eventual, EventualExt}; + use indexer_common::allocations::Allocation; + use indexer_common::escrow_accounts::EscrowAccounts; + use indexer_common::prelude::{DeploymentDetails, SubgraphClient}; + use ractor::concurrency::JoinHandle; + use ractor::{Actor, ActorProcessingErr, ActorRef}; + use sqlx::postgres::PgListener; + use sqlx::PgPool; + use std::collections::{HashMap, HashSet}; + use std::time::Duration; + + const DUMMY_URL: &str = "http://localhost:1234"; + + fn get_subgraph_client() -> &'static SubgraphClient { + Box::leak(Box::new(SubgraphClient::new( + reqwest::Client::new(), + None, + DeploymentDetails::for_query_url(DUMMY_URL).unwrap(), + ))) + } + + fn get_config() -> &'static config::Cli { + Box::leak(Box::new(config::Cli { + config: None, + ethereum: config::Ethereum { + indexer_address: INDEXER.1, + }, + tap: config::Tap { + rav_request_trigger_value: 100, + rav_request_timestamp_buffer_ms: 1, + ..Default::default() + }, + ..Default::default() + })) + } + + async fn create_sender_accounts_manager( + pgpool: PgPool, + ) -> ( + String, + (ActorRef, JoinHandle<()>), + ) { + let config = get_config(); + + let (mut indexer_allocations_writer, indexer_allocations_eventual) = + Eventual::>::new(); + indexer_allocations_writer.write(HashMap::new()); + let escrow_subgraph = get_subgraph_client(); + + let (mut escrow_accounts_writer, escrow_accounts_eventual) = + Eventual::::new(); + escrow_accounts_writer.write(EscrowAccounts::default()); + + let prefix = format!( + "test-{}", + PREFIX_ID.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + ); + let args = SenderAccountsManagerArgs { + config, + domain_separator: TAP_EIP712_DOMAIN_SEPARATOR.clone(), + pgpool, + indexer_allocations: indexer_allocations_eventual, + escrow_accounts: escrow_accounts_eventual, + escrow_subgraph, + sender_aggregator_endpoints: HashMap::from([ + (SENDER.1, String::from("http://localhost:8000")), + (SENDER_2.1, String::from("http://localhost:8000")), + ]), + prefix: Some(prefix.clone()), + }; + ( + prefix, + SenderAccountsManager::spawn(None, SenderAccountsManager, args) + .await + .unwrap(), + ) + } + + #[sqlx::test(migrations = "../migrations")] + async fn test_create_sender_accounts_manager(pgpool: PgPool) { + let (_, (actor, join_handle)) = create_sender_accounts_manager(pgpool).await; + actor.stop_and_wait(None, None).await.unwrap(); + join_handle.await.unwrap(); + } + + fn create_state(pgpool: PgPool) -> (String, State) { + let config = get_config(); + let senders_to_signers = vec![(SENDER.1, vec![SIGNER.1])].into_iter().collect(); + let escrow_accounts = EscrowAccounts::new(HashMap::new(), senders_to_signers); + + let prefix = format!( + "test-{}", + PREFIX_ID.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + ); + ( + prefix.clone(), + State { + config, + domain_separator: TAP_EIP712_DOMAIN_SEPARATOR.clone(), + sender_ids: HashSet::new(), + new_receipts_watcher_handle: tokio::spawn(async {}), + _eligible_allocations_senders_pipe: Eventual::from_value(()) + .pipe_async(|_| async {}), + pgpool, + indexer_allocations: Eventual::from_value(HashSet::new()), + escrow_accounts: Eventual::from_value(escrow_accounts), + escrow_subgraph: get_subgraph_client(), + sender_aggregator_endpoints: HashMap::from([ + (SENDER.1, String::from("http://localhost:8000")), + (SENDER_2.1, String::from("http://localhost:8000")), + ]), + prefix: Some(prefix), + }, + ) + } + + #[sqlx::test(migrations = "../migrations")] + async fn test_pending_sender_allocations(pgpool: PgPool) { + let (_, state) = create_state(pgpool.clone()); + + // add receipts to the database + for i in 1..=10 { + let receipt = create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i, i.into()); + store_receipt(&pgpool, receipt.signed_receipt()) + .await + .unwrap(); + } + + // add non-final ravs + let signed_rav = create_rav(*ALLOCATION_ID_1, SIGNER.0.clone(), 4, 10); + store_rav(&pgpool, signed_rav, SENDER.1).await.unwrap(); + + let pending_allocation_id = state.get_pending_sender_allocation_id().await; + + // check if pending allocations are correct + assert_eq!(pending_allocation_id.len(), 1); + assert!(pending_allocation_id.get(&SENDER.1).is_some()); + assert_eq!(pending_allocation_id.get(&SENDER.1).unwrap().len(), 2); + } + + #[sqlx::test(migrations = "../migrations")] + async fn test_update_sender_allocation(pgpool: PgPool) { + let (prefix, (actor, join_handle)) = create_sender_accounts_manager(pgpool).await; + + actor + .cast(SenderAccountsManagerMessage::UpdateSenderAccounts( + vec![SENDER.1].into_iter().collect(), + )) + .unwrap(); + + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + + // verify if create sender account + let actor_ref = + ActorRef::::where_is(format!("{}:{}", prefix.clone(), SENDER.1)); + assert!(actor_ref.is_some()); + + actor + .cast(SenderAccountsManagerMessage::UpdateSenderAccounts( + HashSet::new(), + )) + .unwrap(); + + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + // verify if it gets removed + let actor_ref = + ActorRef::::where_is(format!("{}:{}", prefix, SENDER.1)); + assert!(actor_ref.is_none()); + + // safely stop the manager + actor.stop_and_wait(None, None).await.unwrap(); + join_handle.await.unwrap(); + } + + #[sqlx::test(migrations = "../migrations")] + async fn test_create_sender_account(pgpool: PgPool) { + struct DummyActor; + #[async_trait::async_trait] + impl Actor for DummyActor { + type Msg = (); + type State = (); + type Arguments = (); + + async fn pre_start( + &self, + _: ActorRef, + _: Self::Arguments, + ) -> Result { + Ok(()) + } + } + + let (prefix, state) = create_state(pgpool.clone()); + let (supervisor, handle) = DummyActor::spawn(None, DummyActor, ()).await.unwrap(); + // we wait to check if the sender is created + + state + .create_sender_account(supervisor.get_cell(), SENDER_2.1, HashSet::new()) + .await + .unwrap(); + + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + + let actor_ref = + ActorRef::::where_is(format!("{}:{}", prefix, SENDER_2.1)); + assert!(actor_ref.is_some()); + + supervisor.stop_and_wait(None, None).await.unwrap(); + handle.await.unwrap(); + } + + #[sqlx::test(migrations = "../migrations")] + async fn test_receive_notifications_(pgpool: PgPool) { + let prefix = format!( + "test-{}", + PREFIX_ID.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + ); + // create dummy allocation + + let (mock_sender_allocation, receipts) = MockSenderAllocation::new_with_receipts(); + let _ = MockSenderAllocation::spawn( + Some(format!( + "{}:{}:{}", + prefix.clone(), + SENDER.1, + *ALLOCATION_ID_0 + )), + mock_sender_allocation, + (), + ) + .await + .unwrap(); + + // create tokio task to listen for notifications + + let mut pglistener = PgListener::connect_with(&pgpool.clone()).await.unwrap(); + pglistener + .listen("scalar_tap_receipt_notification") + .await + .expect( + "should be able to subscribe to Postgres Notify events on the channel \ + 'scalar_tap_receipt_notification'", + ); + + let escrow_accounts_eventual = Eventual::from_value(EscrowAccounts::new( + HashMap::from([(SENDER.1, 1000.into())]), + HashMap::from([(SENDER.1, vec![SIGNER.1])]), + )); + + // Start the new_receipts_watcher task that will consume from the `pglistener` + let new_receipts_watcher_handle = tokio::spawn(new_receipts_watcher( + pglistener, + escrow_accounts_eventual, + Some(prefix.clone()), + )); + + // add receipts to the database + for i in 1..=10 { + let receipt = create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i, i.into()); + store_receipt(&pgpool, receipt.signed_receipt()) + .await + .unwrap(); + } + + tokio::time::sleep(Duration::from_millis(100)).await; + + // check if receipt notification was sent to the allocation + let receipts = receipts.lock().unwrap(); + assert_eq!(receipts.len(), 10); + for (i, receipt) in receipts.iter().enumerate() { + assert_eq!((i + 1) as u64, receipt.id); + } + + new_receipts_watcher_handle.abort(); + } +} diff --git a/tap-agent/src/agent/sender_allocation.rs b/tap-agent/src/agent/sender_allocation.rs new file mode 100644 index 000000000..c06b23a30 --- /dev/null +++ b/tap-agent/src/agent/sender_allocation.rs @@ -0,0 +1,1123 @@ +// Copyright 2023-, GraphOps and Semiotic Labs. +// SPDX-License-Identifier: Apache-2.0 + +use std::{sync::Arc, time::Duration}; + +use alloy_primitives::hex::ToHex; +use alloy_sol_types::Eip712Domain; +use anyhow::{anyhow, ensure, Result}; +use bigdecimal::num_bigint::BigInt; +use eventuals::Eventual; +use indexer_common::{escrow_accounts::EscrowAccounts, prelude::SubgraphClient}; +use jsonrpsee::{core::client::ClientT, http_client::HttpClientBuilder, rpc_params}; +use ractor::{Actor, ActorProcessingErr, ActorRef, RpcReplyPort}; +use sqlx::{types::BigDecimal, PgPool}; +use tap_aggregator::jsonrpsee_helpers::JsonRpcResponse; +use tap_core::{ + rav::{RAVRequest, ReceiptAggregateVoucher}, + receipt::{ + checks::{Check, Checks}, + Failed, ReceiptWithState, + }, + signed_message::EIP712SignedMessage, +}; +use thegraph::types::Address; +use tracing::{error, warn}; + +use crate::agent::sender_account::SenderAccountMessage; +use crate::agent::sender_accounts_manager::NewReceiptNotification; +use crate::agent::unaggregated_receipts::UnaggregatedReceipts; +use crate::{ + config::{self}, + tap::context::{checks::Signature, TapAgentContext}, + tap::signers_trimmed, + tap::{context::checks::AllocationId, escrow_adapter::EscrowAdapter}, +}; + +type TapManager = tap_core::manager::Manager; + +/// Manages unaggregated fees and the TAP lifecyle for a specific (allocation, sender) pair. +pub struct SenderAllocation; + +pub struct SenderAllocationState { + unaggregated_fees: UnaggregatedReceipts, + pgpool: PgPool, + tap_manager: TapManager, + allocation_id: Address, + sender: Address, + sender_aggregator_endpoint: String, + config: &'static config::Cli, + escrow_accounts: Eventual, + domain_separator: Eip712Domain, + sender_account_ref: ActorRef, +} + +pub struct SenderAllocationArgs { + pub config: &'static config::Cli, + pub pgpool: PgPool, + pub allocation_id: Address, + pub sender: Address, + pub escrow_accounts: Eventual, + pub escrow_subgraph: &'static SubgraphClient, + pub escrow_adapter: EscrowAdapter, + pub domain_separator: Eip712Domain, + pub sender_aggregator_endpoint: String, + pub sender_account_ref: ActorRef, +} + +#[derive(Debug)] +pub enum SenderAllocationMessage { + NewReceipt(NewReceiptNotification), + TriggerRAVRequest(RpcReplyPort), + #[cfg(test)] + GetUnaggregatedReceipts(RpcReplyPort), +} + +#[async_trait::async_trait] +impl Actor for SenderAllocation { + type Msg = SenderAllocationMessage; + type State = SenderAllocationState; + type Arguments = SenderAllocationArgs; + + async fn pre_start( + &self, + _myself: ActorRef, + args: Self::Arguments, + ) -> std::result::Result { + let sender_account_ref = args.sender_account_ref.clone(); + let allocation_id = args.allocation_id; + let mut state = SenderAllocationState::new(args); + + state.unaggregated_fees = state.calculate_unaggregated_fee().await?; + sender_account_ref.cast(SenderAccountMessage::UpdateReceiptFees( + allocation_id, + state.unaggregated_fees.clone(), + ))?; + + tracing::info!( + sender = %state.sender, + allocation_id = %state.allocation_id, + "SenderAllocation created!", + ); + + Ok(state) + } + + // this method only runs on graceful stop (real close allocation) + // if the actor crashes, this is not ran + async fn post_stop( + &self, + _myself: ActorRef, + state: &mut Self::State, + ) -> std::result::Result<(), ActorProcessingErr> { + tracing::info!( + sender = %state.sender, + allocation_id = %state.allocation_id, + "Closing SenderAllocation, triggering last rav", + ); + + // Request a RAV and mark the allocation as final. + if state.unaggregated_fees.value > 0 { + state.rav_requester_single().await.inspect_err(|e| { + error!( + "Error while requesting RAV for sender {} and allocation {}: {}", + state.sender, state.allocation_id, e + ); + })?; + } + state.mark_rav_last().await.inspect_err(|e| { + error!( + "Error while marking allocation {} as final for sender {}: {}", + state.allocation_id, state.sender, e + ); + })?; + + Ok(()) + } + + async fn handle( + &self, + _myself: ActorRef, + message: Self::Msg, + state: &mut Self::State, + ) -> std::result::Result<(), ActorProcessingErr> { + tracing::trace!( + sender = %state.sender, + allocation_id = %state.allocation_id, + ?message, + "New SenderAllocation message" + ); + let unaggreated_fees = &mut state.unaggregated_fees; + match message { + SenderAllocationMessage::NewReceipt(NewReceiptNotification { + id, value: fees, .. + }) => { + if id > unaggreated_fees.last_id { + unaggreated_fees.last_id = id; + unaggreated_fees.value = + unaggreated_fees.value.checked_add(fees).unwrap_or_else(|| { + // This should never happen, but if it does, we want to know about it. + error!( + "Overflow when adding receipt value {} to total unaggregated fees {} \ + for allocation {} and sender {}. Setting total unaggregated fees to \ + u128::MAX.", + fees, unaggreated_fees.value, state.allocation_id, state.sender + ); + u128::MAX + }); + state + .sender_account_ref + .cast(SenderAccountMessage::UpdateReceiptFees( + state.allocation_id, + unaggreated_fees.clone(), + ))?; + } + } + // we use a blocking call here to ensure that only one RAV request is running at a time. + SenderAllocationMessage::TriggerRAVRequest(reply) => { + if state.unaggregated_fees.value > 0 { + match state.rav_requester_single().await { + Ok(_) => { + state.unaggregated_fees = state.calculate_unaggregated_fee().await?; + } + Err(e) => { + error! ( + %state.sender, + %state.allocation_id, + error = %e, + "Error while requesting RAV ", + ); + } + } + } + if !reply.is_closed() { + let _ = reply.send(state.unaggregated_fees.clone()); + } + } + #[cfg(test)] + SenderAllocationMessage::GetUnaggregatedReceipts(reply) => { + if !reply.is_closed() { + let _ = reply.send(unaggreated_fees.clone()); + } + } + } + Ok(()) + } +} + +impl SenderAllocationState { + fn new( + SenderAllocationArgs { + config, + pgpool, + allocation_id, + sender, + escrow_accounts, + escrow_subgraph, + escrow_adapter, + domain_separator, + sender_aggregator_endpoint, + sender_account_ref, + }: SenderAllocationArgs, + ) -> Self { + let required_checks: Vec> = vec![ + Arc::new(AllocationId::new( + sender, + allocation_id, + escrow_subgraph, + config, + )), + Arc::new(Signature::new( + domain_separator.clone(), + escrow_accounts.clone(), + )), + ]; + let context = TapAgentContext::new( + pgpool.clone(), + allocation_id, + sender, + escrow_accounts.clone(), + escrow_adapter, + ); + let tap_manager = TapManager::new( + domain_separator.clone(), + context, + Checks::new(required_checks), + ); + + Self { + pgpool, + tap_manager, + allocation_id, + sender, + sender_aggregator_endpoint, + config, + escrow_accounts, + domain_separator, + sender_account_ref: sender_account_ref.clone(), + unaggregated_fees: UnaggregatedReceipts::default(), + } + } + + /// Delete obsolete receipts in the DB w.r.t. the last RAV in DB, then update the tap manager + /// with the latest unaggregated fees from the database. + async fn calculate_unaggregated_fee(&self) -> Result { + tracing::trace!("calculate_unaggregated_fee()"); + self.tap_manager.remove_obsolete_receipts().await?; + + let signers = signers_trimmed(&self.escrow_accounts, self.sender).await?; + + // TODO: Get `rav.timestamp_ns` from the TAP Manager's RAV storage adapter instead? + let res = sqlx::query!( + r#" + WITH rav AS ( + SELECT + timestamp_ns + FROM + scalar_tap_ravs + WHERE + allocation_id = $1 + AND sender_address = $2 + ) + SELECT + MAX(id), + SUM(value) + FROM + scalar_tap_receipts + WHERE + allocation_id = $1 + AND signer_address IN (SELECT unnest($3::text[])) + AND CASE WHEN ( + SELECT + timestamp_ns :: NUMERIC + FROM + rav + ) IS NOT NULL THEN timestamp_ns > ( + SELECT + timestamp_ns :: NUMERIC + FROM + rav + ) ELSE TRUE END + "#, + self.allocation_id.encode_hex::(), + self.sender.encode_hex::(), + &signers + ) + .fetch_one(&self.pgpool) + .await?; + + ensure!( + res.sum.is_none() == res.max.is_none(), + "Exactly one of SUM(value) and MAX(id) is null. This should not happen." + ); + + Ok(UnaggregatedReceipts { + last_id: res.max.unwrap_or(0).try_into()?, + value: res + .sum + .unwrap_or(BigDecimal::from(0)) + .to_string() + .parse::()?, + }) + } + + /// Request a RAV from the sender's TAP aggregator. Only one RAV request will be running at a + /// time through the use of an internal guard. + async fn rav_requester_single(&self) -> Result<()> { + tracing::trace!("rav_requester_single()"); + let RAVRequest { + valid_receipts, + previous_rav, + invalid_receipts, + expected_rav, + } = self + .tap_manager + .create_rav_request( + self.config.tap.rav_request_timestamp_buffer_ms * 1_000_000, + // TODO: limit the number of receipts to aggregate per request. + None, + ) + .await + .map_err(|e| match e { + tap_core::Error::NoValidReceiptsForRAVRequest => anyhow!( + "It looks like there are no valid receipts for the RAV request.\ + This may happen if your `rav_request_trigger_value` is too low \ + and no receipts were found outside the `rav_request_timestamp_buffer_ms`.\ + You can fix this by increasing the `rav_request_trigger_value`." + ), + _ => e.into(), + })?; + if !invalid_receipts.is_empty() { + warn!( + "Found {} invalid receipts for allocation {} and sender {}.", + invalid_receipts.len(), + self.allocation_id, + self.sender + ); + + // Save invalid receipts to the database for logs. + // TODO: consider doing that in a spawned task? + Self::store_invalid_receipts(self, invalid_receipts.as_slice()).await?; + } + let client = HttpClientBuilder::default() + .request_timeout(Duration::from_secs( + self.config.tap.rav_request_timeout_secs, + )) + .build(&self.sender_aggregator_endpoint)?; + let response: JsonRpcResponse> = client + .request( + "aggregate_receipts", + rpc_params!( + "0.0", // TODO: Set the version in a smarter place. + valid_receipts, + previous_rav + ), + ) + .await?; + if let Some(warnings) = response.warnings { + warn!("Warnings from sender's TAP aggregator: {:?}", warnings); + } + match self + .tap_manager + .verify_and_store_rav(expected_rav.clone(), response.data.clone()) + .await + { + Ok(_) => {} + + // Adapter errors are local software errors. Shouldn't be a problem with the sender. + Err(tap_core::Error::AdapterError { source_error: e }) => { + anyhow::bail!("TAP Adapter error while storing RAV: {:?}", e) + } + + // The 3 errors below signal an invalid RAV, which should be about problems with the + // sender. The sender could be malicious. + Err( + e @ tap_core::Error::InvalidReceivedRAV { + expected_rav: _, + received_rav: _, + } + | e @ tap_core::Error::SignatureError(_) + | e @ tap_core::Error::InvalidRecoveredSigner { address: _ }, + ) => { + Self::store_failed_rav(self, &expected_rav, &response.data, &e.to_string()).await?; + anyhow::bail!("Invalid RAV, sender could be malicious: {:?}.", e); + } + + // All relevant errors should be handled above. If we get here, we forgot to handle + // an error case. + Err(e) => { + anyhow::bail!("Error while verifying and storing RAV: {:?}", e); + } + } + Ok(()) + } + + pub async fn mark_rav_last(&self) -> Result<()> { + tracing::info!( + sender = %self.sender, + allocation_id = %self.allocation_id, + "Marking rav as last!", + ); + let updated_rows = sqlx::query!( + r#" + UPDATE scalar_tap_ravs + SET last = true + WHERE allocation_id = $1 AND sender_address = $2 + "#, + self.allocation_id.encode_hex::(), + self.sender.encode_hex::(), + ) + .execute(&self.pgpool) + .await?; + + match updated_rows.rows_affected() { + // in case no rav was marked as final + 0 => { + warn!( + "No RAVs were updated as last for allocation {} and sender {}.", + self.allocation_id, self.sender + ); + Ok(()) + } + 1 => Ok(()), + _ => anyhow::bail!( + "Expected exactly one row to be updated in the latest RAVs table, \ + but {} were updated.", + updated_rows.rows_affected() + ), + } + } + + async fn store_invalid_receipts(&self, receipts: &[ReceiptWithState]) -> Result<()> { + for received_receipt in receipts.iter() { + let receipt = received_receipt.signed_receipt(); + let allocation_id = receipt.message.allocation_id; + let encoded_signature = receipt.signature.to_vec(); + + let receipt_signer = receipt + .recover_signer(&self.domain_separator) + .map_err(|e| { + error!("Failed to recover receipt signer: {}", e); + anyhow!(e) + })?; + + sqlx::query!( + r#" + INSERT INTO scalar_tap_receipts_invalid ( + signer_address, + signature, + allocation_id, + timestamp_ns, + nonce, + value + ) + VALUES ($1, $2, $3, $4, $5, $6) + "#, + receipt_signer.encode_hex::(), + encoded_signature, + allocation_id.encode_hex::(), + BigDecimal::from(receipt.message.timestamp_ns), + BigDecimal::from(receipt.message.nonce), + BigDecimal::from(BigInt::from(receipt.message.value)), + ) + .execute(&self.pgpool) + .await + .map_err(|e| anyhow!("Failed to store failed receipt: {:?}", e))?; + } + + Ok(()) + } + + async fn store_failed_rav( + &self, + expected_rav: &ReceiptAggregateVoucher, + rav: &EIP712SignedMessage, + reason: &str, + ) -> Result<()> { + sqlx::query!( + r#" + INSERT INTO scalar_tap_rav_requests_failed ( + allocation_id, + sender_address, + expected_rav, + rav_response, + reason + ) + VALUES ($1, $2, $3, $4, $5) + "#, + self.allocation_id.encode_hex::(), + self.sender.encode_hex::(), + serde_json::to_value(expected_rav)?, + serde_json::to_value(rav)?, + reason + ) + .execute(&self.pgpool) + .await + .map_err(|e| anyhow!("Failed to store failed RAV: {:?}", e))?; + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::{ + SenderAllocation, SenderAllocationArgs, SenderAllocationMessage, SenderAllocationState, + }; + use crate::{ + agent::{ + sender_account::SenderAccountMessage, sender_accounts_manager::NewReceiptNotification, + unaggregated_receipts::UnaggregatedReceipts, + }, + config, + tap::{ + escrow_adapter::EscrowAdapter, + test_utils::{ + create_rav, create_received_receipt, store_rav, store_receipt, ALLOCATION_ID_0, + INDEXER, SENDER, SIGNER, TAP_EIP712_DOMAIN_SEPARATOR, + }, + }, + }; + use eventuals::Eventual; + use futures::future::join_all; + use indexer_common::{ + escrow_accounts::EscrowAccounts, + subgraph_client::{DeploymentDetails, SubgraphClient}, + }; + use ractor::{ + call, cast, concurrency::JoinHandle, Actor, ActorProcessingErr, ActorRef, ActorStatus, + }; + use serde_json::json; + use sqlx::PgPool; + use std::{ + collections::HashMap, + sync::{Arc, Mutex}, + }; + use tap_aggregator::{jsonrpsee_helpers::JsonRpcResponse, server::run_server}; + use tap_core::receipt::{ + checks::{Check, Checks}, + Checking, ReceiptWithState, + }; + use wiremock::{ + matchers::{body_string_contains, method}, + Mock, MockServer, Respond, ResponseTemplate, + }; + + const DUMMY_URL: &str = "http://localhost:1234"; + + struct MockSenderAccount { + last_message_emitted: Arc>>, + } + + #[async_trait::async_trait] + impl Actor for MockSenderAccount { + type Msg = SenderAccountMessage; + type State = (); + type Arguments = (); + + async fn pre_start( + &self, + _myself: ActorRef, + _allocation_ids: Self::Arguments, + ) -> std::result::Result { + Ok(()) + } + + async fn handle( + &self, + _myself: ActorRef, + message: Self::Msg, + _state: &mut Self::State, + ) -> std::result::Result<(), ActorProcessingErr> { + self.last_message_emitted.lock().unwrap().push(message); + Ok(()) + } + } + + async fn create_mock_sender_account() -> ( + Arc>>, + ActorRef, + JoinHandle<()>, + ) { + let last_message_emitted = Arc::new(Mutex::new(vec![])); + + let (sender_account, join_handle) = MockSenderAccount::spawn( + None, + MockSenderAccount { + last_message_emitted: last_message_emitted.clone(), + }, + (), + ) + .await + .unwrap(); + (last_message_emitted, sender_account, join_handle) + } + + async fn create_sender_allocation_args( + pgpool: PgPool, + sender_aggregator_endpoint: String, + escrow_subgraph_endpoint: &str, + sender_account: Option>, + ) -> SenderAllocationArgs { + let config = Box::leak(Box::new(config::Cli { + config: None, + ethereum: config::Ethereum { + indexer_address: INDEXER.1, + }, + tap: config::Tap { + rav_request_trigger_value: 100, + rav_request_timestamp_buffer_ms: 1, + rav_request_timeout_secs: 5, + ..Default::default() + }, + ..Default::default() + })); + + let escrow_subgraph = Box::leak(Box::new(SubgraphClient::new( + reqwest::Client::new(), + None, + DeploymentDetails::for_query_url(escrow_subgraph_endpoint).unwrap(), + ))); + + let escrow_accounts_eventual = Eventual::from_value(EscrowAccounts::new( + HashMap::from([(SENDER.1, 1000.into())]), + HashMap::from([(SENDER.1, vec![SIGNER.1])]), + )); + + let escrow_adapter = EscrowAdapter::new(escrow_accounts_eventual.clone(), SENDER.1); + + let sender_account_ref = match sender_account { + Some(sender) => sender, + None => create_mock_sender_account().await.1, + }; + + SenderAllocationArgs { + config, + pgpool: pgpool.clone(), + allocation_id: *ALLOCATION_ID_0, + sender: SENDER.1, + escrow_accounts: escrow_accounts_eventual, + escrow_subgraph, + escrow_adapter, + domain_separator: TAP_EIP712_DOMAIN_SEPARATOR.clone(), + sender_aggregator_endpoint, + sender_account_ref, + } + } + + async fn create_sender_allocation( + pgpool: PgPool, + sender_aggregator_endpoint: String, + escrow_subgraph_endpoint: &str, + sender_account: Option>, + ) -> ActorRef { + let args = create_sender_allocation_args( + pgpool, + sender_aggregator_endpoint, + escrow_subgraph_endpoint, + sender_account, + ) + .await; + + let (allocation_ref, _join_handle) = SenderAllocation::spawn(None, SenderAllocation, args) + .await + .unwrap(); + + allocation_ref + } + + #[sqlx::test(migrations = "../migrations")] + async fn should_update_unaggregated_fees_on_start(pgpool: PgPool) { + let (last_message_emitted, sender_account, _join_handle) = + create_mock_sender_account().await; + // Add receipts to the database. + for i in 1..=10 { + let receipt = create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i, i.into()); + store_receipt(&pgpool, receipt.signed_receipt()) + .await + .unwrap(); + } + + let sender_allocation = create_sender_allocation( + pgpool.clone(), + DUMMY_URL.to_string(), + DUMMY_URL, + Some(sender_account), + ) + .await; + + // Get total_unaggregated_fees + let total_unaggregated_fees = call!( + sender_allocation, + SenderAllocationMessage::GetUnaggregatedReceipts + ) + .unwrap(); + + // Should emit a message to the sender account with the unaggregated fees. + let expected_message = SenderAccountMessage::UpdateReceiptFees( + *ALLOCATION_ID_0, + UnaggregatedReceipts { + last_id: 10, + value: 55u128, + }, + ); + let last_message_emitted = last_message_emitted.lock().unwrap(); + assert_eq!(last_message_emitted.len(), 1); + assert_eq!(last_message_emitted.last(), Some(&expected_message)); + + // Check that the unaggregated fees are correct. + assert_eq!(total_unaggregated_fees.value, 55u128); + } + + #[sqlx::test(migrations = "../migrations")] + async fn test_receive_new_receipt(pgpool: PgPool) { + let (last_message_emitted, sender_account, _join_handle) = + create_mock_sender_account().await; + + let sender_allocation = create_sender_allocation( + pgpool.clone(), + DUMMY_URL.to_string(), + DUMMY_URL, + Some(sender_account), + ) + .await; + + // should validate with id less than last_id + cast!( + sender_allocation, + SenderAllocationMessage::NewReceipt(NewReceiptNotification { + id: 0, + value: 10, + allocation_id: *ALLOCATION_ID_0, + signer_address: SIGNER.1, + timestamp_ns: 0, + }) + ) + .unwrap(); + + cast!( + sender_allocation, + SenderAllocationMessage::NewReceipt(NewReceiptNotification { + id: 1, + value: 20, + allocation_id: *ALLOCATION_ID_0, + signer_address: SIGNER.1, + timestamp_ns: 0, + }) + ) + .unwrap(); + + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + + // should emit update aggregate fees message to sender account + let expected_message = SenderAccountMessage::UpdateReceiptFees( + *ALLOCATION_ID_0, + UnaggregatedReceipts { + last_id: 1, + value: 20, + }, + ); + let last_message_emitted = last_message_emitted.lock().unwrap(); + assert_eq!(last_message_emitted.len(), 2); + assert_eq!(last_message_emitted.last(), Some(&expected_message)); + } + + #[sqlx::test(migrations = "../migrations")] + async fn test_trigger_rav_request(pgpool: PgPool) { + // Start a TAP aggregator server. + let (handle, aggregator_endpoint) = run_server( + 0, + SIGNER.0.clone(), + vec![SIGNER.1].into_iter().collect(), + TAP_EIP712_DOMAIN_SEPARATOR.clone(), + 100 * 1024, + 100 * 1024, + 1, + ) + .await + .unwrap(); + + // Start a mock graphql server using wiremock + let mock_server = MockServer::start().await; + + // Mock result for TAP redeem txs for (allocation, sender) pair. + mock_server + .register( + Mock::given(method("POST")) + .and(body_string_contains("transactions")) + .respond_with( + ResponseTemplate::new(200) + .set_body_json(json!({ "data": { "transactions": []}})), + ), + ) + .await; + + // Add receipts to the database. + for i in 0..10 { + let receipt = create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i + 1, i.into()); + store_receipt(&pgpool, receipt.signed_receipt()) + .await + .unwrap(); + } + + // Create a sender_allocation. + let sender_allocation = create_sender_allocation( + pgpool.clone(), + "http://".to_owned() + &aggregator_endpoint.to_string(), + &mock_server.uri(), + None, + ) + .await; + + // Trigger a RAV request manually and wait for updated fees. + let total_unaggregated_fees = call!( + sender_allocation, + SenderAllocationMessage::TriggerRAVRequest + ) + .unwrap(); + + // Check that the unaggregated fees are correct. + assert_eq!(total_unaggregated_fees.value, 0u128); + + // Stop the TAP aggregator server. + handle.stop().unwrap(); + handle.stopped().await; + } + + #[sqlx::test(migrations = "../migrations")] + async fn test_close_allocation_no_pending_fees(pgpool: PgPool) { + let (last_message_emitted, sender_account, _join_handle) = + create_mock_sender_account().await; + + // create allocation + let sender_allocation = create_sender_allocation( + pgpool.clone(), + DUMMY_URL.to_string(), + DUMMY_URL, + Some(sender_account), + ) + .await; + + sender_allocation.stop_and_wait(None, None).await.unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + + // check if the actor is actually stopped + assert_eq!(sender_allocation.get_status(), ActorStatus::Stopped); + + // check if message is sent to sender account + assert_eq!( + last_message_emitted.lock().unwrap().last(), + Some(&SenderAccountMessage::UpdateReceiptFees( + *ALLOCATION_ID_0, + UnaggregatedReceipts::default() + )) + ); + } + + #[sqlx::test(migrations = "../migrations")] + async fn test_close_allocation_with_pending_fees(pgpool: PgPool) { + struct Response { + data: Arc, + } + + impl Respond for Response { + fn respond(&self, _request: &wiremock::Request) -> wiremock::ResponseTemplate { + self.data.notify_one(); + + let mock_rav = create_rav(*ALLOCATION_ID_0, SIGNER.0.clone(), 10, 45); + + let json_response = JsonRpcResponse { + data: mock_rav, + warnings: None, + }; + + ResponseTemplate::new(200).set_body_json(json! ( + { + "id": 0, + "jsonrpc": "2.0", + "result": json_response + } + )) + } + } + + let await_trigger = Arc::new(tokio::sync::Notify::new()); + // Start a TAP aggregator server. + let aggregator_server = MockServer::start().await; + + aggregator_server + .register( + Mock::given(method("POST")) + .and(body_string_contains("aggregate_receipts")) + .respond_with(Response { + data: await_trigger.clone(), + }), + ) + .await; + + // Start a mock graphql server using wiremock + let mock_server = MockServer::start().await; + + // Mock result for TAP redeem txs for (allocation, sender) pair. + mock_server + .register( + Mock::given(method("POST")) + .and(body_string_contains("transactions")) + .respond_with( + ResponseTemplate::new(200) + .set_body_json(json!({ "data": { "transactions": []}})), + ), + ) + .await; + + // Add receipts to the database. + for i in 0..10 { + let receipt = create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i + 1, i.into()); + store_receipt(&pgpool, receipt.signed_receipt()) + .await + .unwrap(); + } + + let (_last_message_emitted, sender_account, _join_handle) = + create_mock_sender_account().await; + + // create allocation + let sender_allocation = create_sender_allocation( + pgpool.clone(), + aggregator_server.uri(), + &mock_server.uri(), + Some(sender_account), + ) + .await; + + sender_allocation.stop_and_wait(None, None).await.unwrap(); + + // should trigger rav request + await_trigger.notified().await; + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + + // check if rav request is made + assert!(aggregator_server.received_requests().await.is_some()); + + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + + // check if the actor is actually stopped + assert_eq!(sender_allocation.get_status(), ActorStatus::Stopped); + } + + #[sqlx::test(migrations = "../migrations")] + async fn should_return_unaggregated_fees_without_rav(pgpool: PgPool) { + let args = + create_sender_allocation_args(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL, None) + .await; + let state = SenderAllocationState::new(args); + + // Add receipts to the database. + for i in 1..10 { + let receipt = create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i, i.into()); + store_receipt(&pgpool, receipt.signed_receipt()) + .await + .unwrap(); + } + + // calculate unaggregated fee + let total_unaggregated_fees = state.calculate_unaggregated_fee().await.unwrap(); + + // Check that the unaggregated fees are correct. + assert_eq!(total_unaggregated_fees.value, 45u128); + } + + /// Test that the sender_allocation correctly updates the unaggregated fees from the + /// database when there is a RAV in the database as well as receipts which timestamp are lesser + /// and greater than the RAV's timestamp. + /// + /// The sender_allocation should only consider receipts with a timestamp greater + /// than the RAV's timestamp. + #[sqlx::test(migrations = "../migrations")] + async fn should_return_unaggregated_fees_with_rav(pgpool: PgPool) { + let args = + create_sender_allocation_args(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL, None) + .await; + let state = SenderAllocationState::new(args); + + // Add the RAV to the database. + // This RAV has timestamp 4. The sender_allocation should only consider receipts + // with a timestamp greater than 4. + let signed_rav = create_rav(*ALLOCATION_ID_0, SIGNER.0.clone(), 4, 10); + store_rav(&pgpool, signed_rav, SENDER.1).await.unwrap(); + + // Add receipts to the database. + for i in 1..10 { + let receipt = create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i, i.into()); + store_receipt(&pgpool, receipt.signed_receipt()) + .await + .unwrap(); + } + + let total_unaggregated_fees = state.calculate_unaggregated_fee().await.unwrap(); + + // Check that the unaggregated fees are correct. + assert_eq!(total_unaggregated_fees.value, 35u128); + } + + #[sqlx::test(migrations = "../migrations")] + async fn test_store_failed_rav(pgpool: PgPool) { + let args = + create_sender_allocation_args(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL, None) + .await; + let state = SenderAllocationState::new(args); + + let signed_rav = create_rav(*ALLOCATION_ID_0, SIGNER.0.clone(), 4, 10); + + // just unit test if it is working + let result = state + .store_failed_rav(&signed_rav.message, &signed_rav, "test") + .await; + + assert!(result.is_ok()); + } + + #[sqlx::test(migrations = "../migrations")] + async fn test_store_invalid_receipts(pgpool: PgPool) { + struct FailingCheck; + + #[async_trait::async_trait] + impl Check for FailingCheck { + async fn check(&self, _receipt: &ReceiptWithState) -> anyhow::Result<()> { + Err(anyhow::anyhow!("Failing check")) + } + } + + let args = + create_sender_allocation_args(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL, None) + .await; + let state = SenderAllocationState::new(args); + + let checks = Checks::new(vec![Arc::new(FailingCheck)]); + + // create some checks + let checking_receipts = vec![ + create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, 1, 1, 1u128), + create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, 2, 2, 2u128), + ]; + // make sure to fail them + let failing_receipts = checking_receipts + .into_iter() + .map(|receipt| async { receipt.finalize_receipt_checks(&checks).await.unwrap_err() }) + .collect::>(); + let failing_receipts = join_all(failing_receipts).await; + + // store the failing receipts + let result = state.store_invalid_receipts(&failing_receipts).await; + + // we just store a few and make sure it doesn't fail + assert!(result.is_ok()); + } + + #[sqlx::test(migrations = "../migrations")] + async fn test_mark_rav_last(pgpool: PgPool) { + let signed_rav = create_rav(*ALLOCATION_ID_0, SIGNER.0.clone(), 4, 10); + store_rav(&pgpool, signed_rav, SENDER.1).await.unwrap(); + + let args = + create_sender_allocation_args(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL, None) + .await; + let state = SenderAllocationState::new(args); + + // mark rav as final + let result = state.mark_rav_last().await; + + // check if it fails + assert!(result.is_ok()); + } + + #[sqlx::test(migrations = "../migrations")] + async fn test_failed_rav_request(pgpool: PgPool) { + // Add receipts to the database. + for i in 0..10 { + let receipt = + create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, u64::max_value(), i.into()); + store_receipt(&pgpool, receipt.signed_receipt()) + .await + .unwrap(); + } + + // Create a sender_allocation. + let sender_allocation = + create_sender_allocation(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL, None).await; + + // Trigger a RAV request manually and wait for updated fees. + // this should fail because there's no receipt with valid timestamp + let total_unaggregated_fees = call!( + sender_allocation, + SenderAllocationMessage::TriggerRAVRequest + ) + .unwrap(); + + // expect the actor to keep running + assert_eq!(sender_allocation.get_status(), ActorStatus::Running); + + // Check that the unaggregated fees return the same value + assert_eq!(total_unaggregated_fees.value, 45u128); + } +} diff --git a/tap-agent/src/agent/sender_fee_tracker.rs b/tap-agent/src/agent/sender_fee_tracker.rs new file mode 100644 index 000000000..2ac5c1684 --- /dev/null +++ b/tap-agent/src/agent/sender_fee_tracker.rs @@ -0,0 +1,109 @@ +// Copyright 2023-, GraphOps and Semiotic Labs. +// SPDX-License-Identifier: Apache-2.0 + +use alloy_primitives::Address; +use std::collections::HashMap; +use tracing::error; + +#[derive(Debug, Clone, Default)] +pub struct SenderFeeTracker { + id_to_fee: HashMap, + total_fee: u128, +} + +impl SenderFeeTracker { + pub fn update(&mut self, id: Address, fee: u128) { + if fee > 0 { + // insert or update, if update remove old fee from total + if let Some(old_fee) = self.id_to_fee.insert(id, fee) { + self.total_fee -= old_fee; + } + self.total_fee = self.total_fee.checked_add(fee).unwrap_or_else(|| { + // This should never happen, but if it does, we want to know about it. + error!( + "Overflow when adding receipt value {} to total fee {}. \ + Setting total fee to u128::MAX.", + fee, self.total_fee + ); + u128::MAX + }); + } else if let Some(old_fee) = self.id_to_fee.remove(&id) { + self.total_fee -= old_fee; + } + } + + pub fn get_heaviest_allocation_id(&self) -> Option
{ + // just loop over and get the biggest fee + self.id_to_fee + .iter() + .fold(None, |acc: Option<(&Address, u128)>, (addr, fee)| { + if let Some((_, max_fee)) = acc { + if *fee > max_fee { + Some((addr, *fee)) + } else { + acc + } + } else { + Some((addr, *fee)) + } + }) + .map(|(&id, _)| id) + } + + pub fn get_total_fee(&self) -> u128 { + self.total_fee + } +} + +#[cfg(test)] +mod tests { + use super::SenderFeeTracker; + use std::str::FromStr; + use thegraph::types::Address; + + #[test] + fn test_allocation_id_tracker() { + let allocation_id_0: Address = + Address::from_str("0xabababababababababababababababababababab").unwrap(); + let allocation_id_1: Address = + Address::from_str("0xbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc").unwrap(); + let allocation_id_2: Address = + Address::from_str("0xcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd").unwrap(); + + let mut tracker = SenderFeeTracker::default(); + assert_eq!(tracker.get_heaviest_allocation_id(), None); + assert_eq!(tracker.get_total_fee(), 0); + + tracker.update(allocation_id_0, 10); + assert_eq!(tracker.get_heaviest_allocation_id(), Some(allocation_id_0)); + assert_eq!(tracker.get_total_fee(), 10); + + tracker.update(allocation_id_2, 20); + assert_eq!(tracker.get_heaviest_allocation_id(), Some(allocation_id_2)); + assert_eq!(tracker.get_total_fee(), 30); + + tracker.update(allocation_id_1, 30); + assert_eq!(tracker.get_heaviest_allocation_id(), Some(allocation_id_1)); + assert_eq!(tracker.get_total_fee(), 60); + + tracker.update(allocation_id_2, 10); + assert_eq!(tracker.get_heaviest_allocation_id(), Some(allocation_id_1)); + assert_eq!(tracker.get_total_fee(), 50); + + tracker.update(allocation_id_2, 40); + assert_eq!(tracker.get_heaviest_allocation_id(), Some(allocation_id_2)); + assert_eq!(tracker.get_total_fee(), 80); + + tracker.update(allocation_id_1, 0); + assert_eq!(tracker.get_heaviest_allocation_id(), Some(allocation_id_2)); + assert_eq!(tracker.get_total_fee(), 50); + + tracker.update(allocation_id_2, 0); + assert_eq!(tracker.get_heaviest_allocation_id(), Some(allocation_id_0)); + assert_eq!(tracker.get_total_fee(), 10); + + tracker.update(allocation_id_0, 0); + assert_eq!(tracker.get_heaviest_allocation_id(), None); + assert_eq!(tracker.get_total_fee(), 0); + } +} diff --git a/tap-agent/src/tap/unaggregated_receipts.rs b/tap-agent/src/agent/unaggregated_receipts.rs similarity index 89% rename from tap-agent/src/tap/unaggregated_receipts.rs rename to tap-agent/src/agent/unaggregated_receipts.rs index 0511152c7..96a410abf 100644 --- a/tap-agent/src/tap/unaggregated_receipts.rs +++ b/tap-agent/src/agent/unaggregated_receipts.rs @@ -1,7 +1,7 @@ // Copyright 2023-, GraphOps and Semiotic Labs. // SPDX-License-Identifier: Apache-2.0 -#[derive(Default, Debug, Clone)] +#[derive(Default, Debug, Clone, Eq, PartialEq)] pub struct UnaggregatedReceipts { pub value: u128, /// The ID of the last receipt value added to the unaggregated fees value. diff --git a/tap-agent/src/lib.rs b/tap-agent/src/lib.rs new file mode 100644 index 000000000..3e8258508 --- /dev/null +++ b/tap-agent/src/lib.rs @@ -0,0 +1,23 @@ +// Copyright 2023-, GraphOps and Semiotic Labs. +// SPDX-License-Identifier: Apache-2.0 + +use alloy_sol_types::{eip712_domain, Eip712Domain}; +use lazy_static::lazy_static; + +use crate::config::Cli; + +lazy_static! { + pub static ref CONFIG: Cli = Cli::args(); + pub static ref EIP_712_DOMAIN: Eip712Domain = eip712_domain! { + name: "TAP", + version: "1", + chain_id: CONFIG.receipts.receipts_verifier_chain_id, + verifying_contract: CONFIG.receipts.receipts_verifier_address, + }; +} + +pub mod agent; +pub mod aggregator_endpoints; +pub mod config; +pub mod database; +pub mod tap; diff --git a/tap-agent/src/main.rs b/tap-agent/src/main.rs index a99ac21df..b53b13a32 100644 --- a/tap-agent/src/main.rs +++ b/tap-agent/src/main.rs @@ -2,21 +2,11 @@ // SPDX-License-Identifier: Apache-2.0 use anyhow::Result; -use lazy_static::lazy_static; +use ractor::ActorStatus; use tokio::signal::unix::{signal, SignalKind}; -use tracing::{debug, info}; +use tracing::{debug, error, info}; -use crate::config::Cli; - -mod agent; -mod aggregator_endpoints; -mod config; -mod database; -mod tap; - -lazy_static! { - pub static ref CONFIG: Cli = Cli::args(); -} +use indexer_tap_agent::{agent, CONFIG}; #[tokio::main] async fn main() -> Result<()> { @@ -24,22 +14,27 @@ async fn main() -> Result<()> { lazy_static::initialize(&CONFIG); debug!("Config: {:?}", *CONFIG); - { - let _manager = agent::start_agent(&CONFIG).await; - info!("TAP Agent started."); + let (manager, handler) = agent::start_agent().await; + info!("TAP Agent started."); - // Have tokio wait for SIGTERM or SIGINT. - let mut signal_sigint = signal(SignalKind::interrupt())?; - let mut signal_sigterm = signal(SignalKind::terminate())?; - tokio::select! { - _ = signal_sigint.recv() => debug!("Received SIGINT."), - _ = signal_sigterm.recv() => debug!("Received SIGTERM."), - } - - // If we're here, we've received a signal to exit. - info!("Shutting down..."); + // Have tokio wait for SIGTERM or SIGINT. + let mut signal_sigint = signal(SignalKind::interrupt())?; + let mut signal_sigterm = signal(SignalKind::terminate())?; + tokio::select! { + _ = handler => error!("SenderAccountsManager stopped"), + _ = signal_sigint.recv() => debug!("Received SIGINT."), + _ = signal_sigterm.recv() => debug!("Received SIGTERM."), + } + // If we're here, we've received a signal to exit. + info!("Shutting down..."); + + // We don't want our actor to run any shutdown logic, so we kill it. + if manager.get_status() == ActorStatus::Running { + manager + .kill_and_wait(None) + .await + .expect("Failed to kill manager."); } - // Manager should be successfully dropped here. // Stop the server and wait for it to finish gracefully. debug!("Goodbye!"); diff --git a/tap-agent/src/tap/context.rs b/tap-agent/src/tap/context.rs index 2cf9c0db4..716330d7a 100644 --- a/tap-agent/src/tap/context.rs +++ b/tap-agent/src/tap/context.rs @@ -21,7 +21,6 @@ pub struct TapAgentContext { allocation_id: Address, sender: Address, escrow_accounts: Eventual, - // sender_pending_fees: Arc>>, escrow_adapter: EscrowAdapter, } @@ -38,7 +37,6 @@ impl TapAgentContext { allocation_id, sender, escrow_accounts, - // sender_pending_fees: Arc::new(RwLock::new(HashMap::new())), escrow_adapter, } } diff --git a/tap-agent/src/tap/context/rav.rs b/tap-agent/src/tap/context/rav.rs index 8979b75c2..c3284ad7a 100644 --- a/tap-agent/src/tap/context/rav.rs +++ b/tap-agent/src/tap/context/rav.rs @@ -154,8 +154,7 @@ mod test { SIGNER.0.clone(), timestamp_ns, value_aggregate, - ) - .await; + ); context.update_last_rav(new_rav.clone()).await.unwrap(); // Should trigger a retrieve_last_rav So eventually the last rav should be the one @@ -170,8 +169,7 @@ mod test { SIGNER.0.clone(), timestamp_ns + i, value_aggregate - (i as u128), - ) - .await; + ); context.update_last_rav(new_rav.clone()).await.unwrap(); } diff --git a/tap-agent/src/tap/context/receipt.rs b/tap-agent/src/tap/context/receipt.rs index cbfc75a6c..179aeff48 100644 --- a/tap-agent/src/tap/context/receipt.rs +++ b/tap-agent/src/tap/context/receipt.rs @@ -184,21 +184,27 @@ impl ReceiptDelete for TapAgentContext { #[cfg(test)] mod test { - use std::collections::HashMap; - use super::*; use crate::tap::{ escrow_adapter::EscrowAdapter, test_utils::{ - create_received_receipt, store_receipt, ALLOCATION_ID_0, ALLOCATION_ID_IRRELEVANT, - SENDER, SENDER_IRRELEVANT, SIGNER, TAP_EIP712_DOMAIN_SEPARATOR, + create_received_receipt, store_receipt, wallet, ALLOCATION_ID_0, SENDER, SIGNER, + TAP_EIP712_DOMAIN_SEPARATOR, }, }; use anyhow::Result; - + use ethers_signers::LocalWallet; use eventuals::Eventual; use indexer_common::escrow_accounts::EscrowAccounts; + use lazy_static::lazy_static; use sqlx::PgPool; + use std::collections::HashMap; + + lazy_static! { + pub static ref SENDER_IRRELEVANT: (LocalWallet, Address) = wallet(1); + pub static ref ALLOCATION_ID_IRRELEVANT: Address = + Address::from_str("0xbcdebcdebcdebcdebcdebcdebcdebcdebcdebcde").unwrap(); + } /// Insert a single receipt and retrieve it from the database using the adapter. /// The point here it to test the deserialization of large numbers. @@ -218,8 +224,7 @@ mod test { ); let received_receipt = - create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, u64::MAX, u64::MAX, u128::MAX) - .await; + create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, u64::MAX, u64::MAX, u128::MAX); // Storing the receipt store_receipt(&storage_adapter.pgpool, received_receipt.signed_receipt()) @@ -437,38 +442,29 @@ mod test { // Creating 10 receipts with timestamps 42 to 51 let mut received_receipt_vec = Vec::new(); for i in 0..10 { - received_receipt_vec.push( - create_received_receipt( - &ALLOCATION_ID_0, - &SIGNER.0, - i + 684, - i + 42, - (i + 124).into(), - ) - .await, - ); + received_receipt_vec.push(create_received_receipt( + &ALLOCATION_ID_0, + &SIGNER.0, + i + 684, + i + 42, + (i + 124).into(), + )); // Adding irrelevant receipts to make sure they are not retrieved - received_receipt_vec.push( - create_received_receipt( - &ALLOCATION_ID_IRRELEVANT, - &SIGNER.0, - i + 684, - i + 42, - (i + 124).into(), - ) - .await, - ); - received_receipt_vec.push( - create_received_receipt( - &ALLOCATION_ID_0, - &SENDER_IRRELEVANT.0, - i + 684, - i + 42, - (i + 124).into(), - ) - .await, - ); + received_receipt_vec.push(create_received_receipt( + &ALLOCATION_ID_IRRELEVANT, + &SIGNER.0, + i + 684, + i + 42, + (i + 124).into(), + )); + received_receipt_vec.push(create_received_receipt( + &ALLOCATION_ID_0, + &SENDER_IRRELEVANT.0, + i + 684, + i + 42, + (i + 124).into(), + )); } // Storing the receipts @@ -574,38 +570,29 @@ mod test { // Creating 10 receipts with timestamps 42 to 51 let mut received_receipt_vec = Vec::new(); for i in 0..10 { - received_receipt_vec.push( - create_received_receipt( - &ALLOCATION_ID_0, - &SIGNER.0, - i + 684, - i + 42, - (i + 124).into(), - ) - .await, - ); + received_receipt_vec.push(create_received_receipt( + &ALLOCATION_ID_0, + &SIGNER.0, + i + 684, + i + 42, + (i + 124).into(), + )); // Adding irrelevant receipts to make sure they are not retrieved - received_receipt_vec.push( - create_received_receipt( - &ALLOCATION_ID_IRRELEVANT, - &SIGNER.0, - i + 684, - i + 42, - (i + 124).into(), - ) - .await, - ); - received_receipt_vec.push( - create_received_receipt( - &ALLOCATION_ID_0, - &SENDER_IRRELEVANT.0, - i + 684, - i + 42, - (i + 124).into(), - ) - .await, - ); + received_receipt_vec.push(create_received_receipt( + &ALLOCATION_ID_IRRELEVANT, + &SIGNER.0, + i + 684, + i + 42, + (i + 124).into(), + )); + received_receipt_vec.push(create_received_receipt( + &ALLOCATION_ID_0, + &SENDER_IRRELEVANT.0, + i + 684, + i + 42, + (i + 124).into(), + )); } macro_rules! test_ranges{ diff --git a/tap-agent/src/tap/mod.rs b/tap-agent/src/tap/mod.rs index bb72d83cc..3802cf8b7 100644 --- a/tap-agent/src/tap/mod.rs +++ b/tap-agent/src/tap/mod.rs @@ -7,17 +7,13 @@ use eventuals::Eventual; use indexer_common::escrow_accounts::EscrowAccounts; use thegraph::types::Address; -mod context; -mod escrow_adapter; -mod sender_account; -pub mod sender_accounts_manager; -mod sender_allocation; -mod unaggregated_receipts; +pub mod context; +pub mod escrow_adapter; #[cfg(test)] pub mod test_utils; -async fn signers_trimmed( +pub async fn signers_trimmed( escrow_accounts: &Eventual, sender: Address, ) -> Result, anyhow::Error> { diff --git a/tap-agent/src/tap/sender_account.rs b/tap-agent/src/tap/sender_account.rs deleted file mode 100644 index 653149f59..000000000 --- a/tap-agent/src/tap/sender_account.rs +++ /dev/null @@ -1,1000 +0,0 @@ -// Copyright 2023-, GraphOps and Semiotic Labs. -// SPDX-License-Identifier: Apache-2.0 - -use std::sync::Mutex as StdMutex; -use std::{ - cmp::max, - collections::{HashMap, HashSet}, - sync::Arc, - time::Duration, -}; - -use alloy_sol_types::Eip712Domain; -use anyhow::{anyhow, Result}; -use enum_as_inner::EnumAsInner; -use eventuals::Eventual; -use indexer_common::{escrow_accounts::EscrowAccounts, prelude::SubgraphClient}; -use sqlx::PgPool; -use thegraph::types::Address; -use tokio::sync::Mutex as TokioMutex; -use tokio::time::sleep; -use tokio::{select, sync::Notify, time}; -use tracing::{error, warn}; - -use crate::config::{self}; -use crate::tap::{ - escrow_adapter::EscrowAdapter, sender_accounts_manager::NewReceiptNotification, - sender_allocation::SenderAllocation, unaggregated_receipts::UnaggregatedReceipts, -}; - -#[derive(Clone, EnumAsInner)] -enum AllocationState { - Active(Arc), - Ineligible(Arc), -} - -/// The inner state of a SenderAccount. This is used to store an Arc state for spawning async tasks. -pub struct Inner { - config: &'static config::Cli, - pgpool: PgPool, - allocations: Arc>>, - sender: Address, - sender_aggregator_endpoint: String, - unaggregated_fees: Arc>, - unaggregated_receipts_guard: Arc>, -} - -/// Wrapper around a `Notify` to trigger RAV requests. -/// This gives a better understanding of what is happening -/// because the methods names are more explicit. -#[derive(Clone)] -pub struct RavTrigger(Arc); - -impl RavTrigger { - pub fn new() -> Self { - Self(Arc::new(Notify::new())) - } - - /// Trigger a RAV request if there are any waiters. - /// In case there are no waiters, nothing happens. - pub fn trigger_rav(&self) { - self.0.notify_waiters(); - } - - /// Wait for a RAV trigger. - pub async fn wait_for_rav_request(&self) { - self.0.notified().await; - } - - /// Trigger a RAV request. In case there are no waiters, the request is queued - /// and is executed on the next call to wait_for_rav_trigger(). - pub fn trigger_next_rav(&self) { - self.0.notify_one(); - } -} - -impl Inner { - async fn rav_requester(&self, trigger: RavTrigger) { - loop { - trigger.wait_for_rav_request().await; - - if let Err(error) = self.rav_requester_single().await { - // If an error occoured, we shouldn't retry right away, so we wait for a bit. - error!( - "Error while requesting RAV for sender {}: {}", - self.sender, error - ); - // simpler for now, maybe we can add a backoff strategy later - sleep(Duration::from_secs(5)).await; - continue; - } - - // Check if we already need to send another RAV request. - let unaggregated_fees = self.unaggregated_fees.lock().unwrap().clone(); - if unaggregated_fees.value >= self.config.tap.rav_request_trigger_value.into() { - // If so, "self-notify" to trigger another RAV request. - trigger.trigger_next_rav(); - - warn!( - "Sender {} has {} unaggregated fees immediately after a RAV request, which is - over the trigger value. Triggering another RAV request.", - self.sender, unaggregated_fees.value, - ); - } - } - } - - async fn rav_requester_finalize(&self, finalize_trigger: RavTrigger) { - loop { - // Wait for either 5 minutes or a notification that we need to try to finalize - // allocation receipts. - select! { - _ = time::sleep(Duration::from_secs(300)) => (), - _ = finalize_trigger.wait_for_rav_request() => () - } - - // Get a quick snapshot of the current finalizing allocations. They are - // Arcs, so it should be cheap. - let allocations_finalizing = self - .allocations - .lock() - .unwrap() - .values() - .filter(|a| matches!(a, AllocationState::Ineligible(_))) - .map(|a| a.as_ineligible().unwrap()) - .cloned() - .collect::>(); - - for allocation in allocations_finalizing { - if let Err(e) = allocation.rav_requester_single().await { - error!( - "Error while requesting RAV for sender {} and allocation {}: {}", - self.sender, - allocation.get_allocation_id(), - e - ); - continue; - } - - if let Err(e) = allocation.mark_rav_last().await { - error!( - "Error while marking allocation {} as last for sender {}: {}", - allocation.get_allocation_id(), - self.sender, - e - ); - continue; - } - - // Remove the allocation from the finalizing map. - self.allocations - .lock() - .unwrap() - .remove(&allocation.get_allocation_id()); - } - } - } - - /// Does a single RAV request for the sender's allocation with the highest unaggregated fees - async fn rav_requester_single(&self) -> Result<()> { - let heaviest_allocation = self.get_heaviest_allocation().ok_or(anyhow! { - "Error while getting allocation with most unaggregated fees", - })?; - heaviest_allocation - .rav_requester_single() - .await - .map_err(|e| { - anyhow! { - "Error while requesting RAV for sender {} and allocation {}: {}", - self.sender, - heaviest_allocation.get_allocation_id(), - e - } - })?; - - self.recompute_unaggregated_fees().await; - - Ok(()) - } - - /// Returns the allocation with the highest unaggregated fees value. - /// If there are no active allocations, returns None. - fn get_heaviest_allocation(&self) -> Option> { - // Get a quick snapshot of all allocations. They are Arcs, so it should be cheap, - // and we don't want to hold the lock for too long. - let allocations: Vec<_> = self.allocations.lock().unwrap().values().cloned().collect(); - - let mut heaviest_allocation = (None, 0u128); - for allocation in allocations { - let allocation: Arc = match allocation { - AllocationState::Active(a) => a, - AllocationState::Ineligible(a) => a, - }; - let fees = allocation.get_unaggregated_fees().value; - if fees > heaviest_allocation.1 { - heaviest_allocation = (Some(allocation), fees); - } - } - - heaviest_allocation.0 - } - - /// Recompute the sender's total unaggregated fees value and last receipt ID. - async fn recompute_unaggregated_fees(&self) { - // Make sure to pause the handling of receipt notifications while we update the unaggregated - // fees. - let _guard = self.unaggregated_receipts_guard.lock().await; - - // Similar pattern to get_heaviest_allocation(). - let allocations: Vec<_> = self.allocations.lock().unwrap().values().cloned().collect(); - - // Gather the unaggregated fees from all allocations and sum them up. - let mut unaggregated_fees = self.unaggregated_fees.lock().unwrap(); - *unaggregated_fees = UnaggregatedReceipts::default(); // Reset to 0. - for allocation in allocations { - let allocation: Arc = match allocation { - AllocationState::Active(a) => a, - AllocationState::Ineligible(a) => a, - }; - - let uf = allocation.get_unaggregated_fees(); - *unaggregated_fees = UnaggregatedReceipts { - value: self.fees_add(unaggregated_fees.value, uf.value), - last_id: max(unaggregated_fees.last_id, uf.last_id), - }; - } - } - - /// Safe add the fees to the unaggregated fees value, log an error if there is an overflow and - /// set the unaggregated fees value to u128::MAX. - fn fees_add(&self, total_unaggregated_fees: u128, value_increment: u128) -> u128 { - total_unaggregated_fees - .checked_add(value_increment) - .unwrap_or_else(|| { - // This should never happen, but if it does, we want to know about it. - error!( - "Overflow when adding receipt value {} to total unaggregated fees {} for \ - sender {}. Setting total unaggregated fees to u128::MAX.", - value_increment, total_unaggregated_fees, self.sender - ); - u128::MAX - }) - } -} - -/// A SenderAccount manages the receipts accounting between the indexer and the sender across -/// multiple allocations. -/// -/// Manages the lifecycle of Scalar TAP for the SenderAccount, including: -/// - Monitoring new receipts and keeping track of the cumulative unaggregated fees across -/// allocations. -/// - Requesting RAVs from the sender's TAP aggregator once the cumulative unaggregated fees reach a -/// certain threshold. -/// - Requesting the last RAV from the sender's TAP aggregator for all EOL allocations. -pub struct SenderAccount { - inner: Arc, - escrow_accounts: Eventual, - escrow_subgraph: &'static SubgraphClient, - escrow_adapter: EscrowAdapter, - tap_eip712_domain_separator: Eip712Domain, - rav_requester_task: tokio::task::JoinHandle<()>, - rav_trigger: RavTrigger, - rav_requester_finalize_task: tokio::task::JoinHandle<()>, - rav_finalize_trigger: RavTrigger, - unaggregated_receipts_guard: Arc>, -} - -impl SenderAccount { - #[allow(clippy::too_many_arguments)] - pub fn new( - config: &'static config::Cli, - pgpool: PgPool, - sender_id: Address, - escrow_accounts: Eventual, - escrow_subgraph: &'static SubgraphClient, - tap_eip712_domain_separator: Eip712Domain, - sender_aggregator_endpoint: String, - ) -> Self { - let unaggregated_receipts_guard = Arc::new(TokioMutex::new(())); - - let escrow_adapter = EscrowAdapter::new(escrow_accounts.clone(), sender_id); - - let inner = Arc::new(Inner { - config, - pgpool, - allocations: Arc::new(StdMutex::new(HashMap::new())), - sender: sender_id, - sender_aggregator_endpoint, - unaggregated_fees: Arc::new(StdMutex::new(UnaggregatedReceipts::default())), - unaggregated_receipts_guard: unaggregated_receipts_guard.clone(), - }); - - let rav_trigger = RavTrigger::new(); - let rav_requester_task = tokio::spawn({ - let inner = inner.clone(); - let rav_trigger = rav_trigger.clone(); - async move { - inner.rav_requester(rav_trigger).await; - } - }); - - let rav_finalize_trigger = RavTrigger::new(); - let rav_requester_finalize_task = tokio::spawn({ - let inner = inner.clone(); - let rav_finalize_trigger = rav_finalize_trigger.clone(); - async move { - inner.rav_requester_finalize(rav_finalize_trigger).await; - } - }); - - Self { - inner: inner.clone(), - escrow_accounts, - escrow_subgraph, - escrow_adapter, - tap_eip712_domain_separator, - rav_requester_task, - rav_trigger, - rav_requester_finalize_task, - rav_finalize_trigger, - unaggregated_receipts_guard, - } - } - - /// Update the sender's allocations to match the target allocations. - pub async fn update_allocations(&self, target_allocations: HashSet
) { - { - let mut allocations = self.inner.allocations.lock().unwrap(); - let mut allocations_to_finalize = false; - - // Make allocations that are no longer to be active `AllocationState::Ineligible`. - for (allocation_id, allocation_state) in allocations.iter_mut() { - if !target_allocations.contains(allocation_id) { - match allocation_state { - AllocationState::Active(allocation) => { - *allocation_state = AllocationState::Ineligible(allocation.clone()); - allocations_to_finalize = true; - } - AllocationState::Ineligible(_) => { - // Allocation is already ineligible, do nothing. - } - } - } - } - - if allocations_to_finalize { - self.rav_finalize_trigger.trigger_rav(); - } - } - - // Add new allocations. - for allocation_id in target_allocations { - let sender_allocation = AllocationState::Active(Arc::new( - SenderAllocation::new( - self.inner.config, - self.inner.pgpool.clone(), - allocation_id, - self.inner.sender, - self.escrow_accounts.clone(), - self.escrow_subgraph, - self.escrow_adapter.clone(), - self.tap_eip712_domain_separator.clone(), - self.inner.sender_aggregator_endpoint.clone(), - ) - .await, - )); - if let std::collections::hash_map::Entry::Vacant(e) = - self.inner.allocations.lock().unwrap().entry(allocation_id) - { - e.insert(sender_allocation); - } - } - } - - pub async fn handle_new_receipt_notification( - &self, - new_receipt_notification: NewReceiptNotification, - ) { - // Make sure to pause the handling of receipt notifications while we update the unaggregated - // fees. - let _guard = self.unaggregated_receipts_guard.lock().await; - - let allocation_state = self - .inner - .allocations - .lock() - .unwrap() - .get(&new_receipt_notification.allocation_id) - .cloned(); - - if let Some(AllocationState::Active(allocation)) = allocation_state { - // Try to add the receipt value to the allocation's unaggregated fees value. - // If the fees were not added, it means the receipt was already processed, so we - // don't need to do anything. - if allocation - .fees_add(new_receipt_notification.value, new_receipt_notification.id) - .await - { - // Add the receipt value to the allocation's unaggregated fees value. - allocation - .fees_add(new_receipt_notification.value, new_receipt_notification.id) - .await; - // Add the receipt value to the sender's unaggregated fees value. - let mut unaggregated_fees = self.inner.unaggregated_fees.lock().unwrap(); - *unaggregated_fees = UnaggregatedReceipts { - value: self - .inner - .fees_add(unaggregated_fees.value, new_receipt_notification.value), - last_id: new_receipt_notification.id, - }; - - // Check if we need to trigger a RAV request. - if unaggregated_fees.value >= self.inner.config.tap.rav_request_trigger_value.into() - { - self.rav_trigger.trigger_rav(); - } - } - } else { - error!( - "Received a new receipt notification for allocation {} that doesn't exist \ - or is ineligible for sender {}.", - new_receipt_notification.allocation_id, self.inner.sender - ); - } - } - - pub async fn recompute_unaggregated_fees(&self) { - self.inner.recompute_unaggregated_fees().await - } -} - -// Abort tasks on Drop -impl Drop for SenderAccount { - fn drop(&mut self) { - self.rav_requester_task.abort(); - self.rav_requester_finalize_task.abort(); - } -} - -#[cfg(test)] -mod tests { - use alloy_primitives::hex::ToHex; - use bigdecimal::{num_bigint::ToBigInt, ToPrimitive}; - use indexer_common::subgraph_client::DeploymentDetails; - use serde_json::json; - use std::str::FromStr; - use tap_aggregator::server::run_server; - use tap_core::{rav::ReceiptAggregateVoucher, signed_message::EIP712SignedMessage}; - use wiremock::{ - matchers::{body_string_contains, method}, - Mock, MockServer, ResponseTemplate, - }; - - use crate::tap::test_utils::{ - create_received_receipt, store_receipt, ALLOCATION_ID_0, ALLOCATION_ID_1, ALLOCATION_ID_2, - INDEXER, SENDER, SIGNER, TAP_EIP712_DOMAIN_SEPARATOR, - }; - - use super::*; - - const DUMMY_URL: &str = "http://localhost:1234"; - - // To help with testing from other modules. - impl SenderAccount { - pub fn _tests_get_allocations_active(&self) -> HashMap> { - self.inner - .allocations - .lock() - .unwrap() - .iter() - .filter_map(|(k, v)| { - if let AllocationState::Active(a) = v { - Some((*k, a.clone())) - } else { - None - } - }) - .collect() - } - - pub fn _tests_get_allocations_ineligible(&self) -> HashMap> { - self.inner - .allocations - .lock() - .unwrap() - .iter() - .filter_map(|(k, v)| { - if let AllocationState::Ineligible(a) = v { - Some((*k, a.clone())) - } else { - None - } - }) - .collect() - } - } - - async fn create_sender_with_allocations( - pgpool: PgPool, - sender_aggregator_endpoint: String, - escrow_subgraph_endpoint: &str, - ) -> SenderAccount { - let config = Box::leak(Box::new(config::Cli { - config: None, - ethereum: config::Ethereum { - indexer_address: INDEXER.1, - }, - tap: config::Tap { - rav_request_trigger_value: 100, - rav_request_timestamp_buffer_ms: 1, - rav_request_timeout_secs: 5, - ..Default::default() - }, - ..Default::default() - })); - - let escrow_subgraph = Box::leak(Box::new(SubgraphClient::new( - reqwest::Client::new(), - None, - DeploymentDetails::for_query_url(escrow_subgraph_endpoint).unwrap(), - ))); - - let escrow_accounts_eventual = Eventual::from_value(EscrowAccounts::new( - HashMap::from([(SENDER.1, 1000.into())]), - HashMap::from([(SENDER.1, vec![SIGNER.1])]), - )); - - let sender = SenderAccount::new( - config, - pgpool, - SENDER.1, - escrow_accounts_eventual, - escrow_subgraph, - TAP_EIP712_DOMAIN_SEPARATOR.clone(), - sender_aggregator_endpoint, - ); - - sender - .update_allocations(HashSet::from([ - *ALLOCATION_ID_0, - *ALLOCATION_ID_1, - *ALLOCATION_ID_2, - ])) - .await; - sender.recompute_unaggregated_fees().await; - - sender - } - - /// Test that the sender_account correctly ignores new receipt notifications with - /// an ID lower than the last receipt ID processed (be it from the DB or from a prior receipt - /// notification). - #[sqlx::test(migrations = "../migrations")] - async fn test_handle_new_receipt_notification(pgpool: PgPool) { - // Add receipts to the database. Before creating the sender and allocation so that it loads - // the receipts from the DB. - let mut expected_unaggregated_fees = 0u128; - for i in 10..20 { - let receipt = - create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i, i.into()).await; - store_receipt(&pgpool, receipt.signed_receipt()) - .await - .unwrap(); - expected_unaggregated_fees += u128::from(i); - } - - let sender = - create_sender_with_allocations(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL).await; - - // Check that the allocation's unaggregated fees are correct. - assert_eq!( - sender - .inner - .allocations - .lock() - .unwrap() - .get(&*ALLOCATION_ID_0) - .unwrap() - .as_active() - .unwrap() - .get_unaggregated_fees() - .value, - expected_unaggregated_fees - ); - - // Check that the sender's unaggregated fees are correct. - assert_eq!( - sender.inner.unaggregated_fees.lock().unwrap().value, - expected_unaggregated_fees - ); - - // Send a new receipt notification that has a lower ID than the last loaded from the DB. - // The last ID in the DB should be 10, since we added 10 receipts to the empty receipts - // table - let new_receipt_notification = NewReceiptNotification { - allocation_id: *ALLOCATION_ID_0, - signer_address: SIGNER.1, - id: 10, - timestamp_ns: 19, - value: 19, - }; - sender - .handle_new_receipt_notification(new_receipt_notification) - .await; - - // Check that the allocation's unaggregated fees have *not* increased. - assert_eq!( - sender - .inner - .allocations - .lock() - .unwrap() - .get(&*ALLOCATION_ID_0) - .unwrap() - .as_active() - .unwrap() - .get_unaggregated_fees() - .value, - expected_unaggregated_fees - ); - - // Check that the unaggregated fees have *not* increased. - assert_eq!( - sender.inner.unaggregated_fees.lock().unwrap().value, - expected_unaggregated_fees - ); - - // Send a new receipt notification. - let new_receipt_notification = NewReceiptNotification { - allocation_id: *ALLOCATION_ID_0, - signer_address: SIGNER.1, - id: 30, - timestamp_ns: 20, - value: 20, - }; - sender - .handle_new_receipt_notification(new_receipt_notification) - .await; - expected_unaggregated_fees += 20; - - // Check that the allocation's unaggregated fees are correct. - assert_eq!( - sender - .inner - .allocations - .lock() - .unwrap() - .get(&*ALLOCATION_ID_0) - .unwrap() - .as_active() - .unwrap() - .get_unaggregated_fees() - .value, - expected_unaggregated_fees - ); - - // Check that the unaggregated fees are correct. - assert_eq!( - sender.inner.unaggregated_fees.lock().unwrap().value, - expected_unaggregated_fees - ); - - // Send a new receipt notification that has a lower ID than the previous one. - let new_receipt_notification = NewReceiptNotification { - allocation_id: *ALLOCATION_ID_0, - signer_address: SIGNER.1, - id: 25, - timestamp_ns: 19, - value: 19, - }; - sender - .handle_new_receipt_notification(new_receipt_notification) - .await; - - // Check that the allocation's unaggregated fees have *not* increased. - assert_eq!( - sender - .inner - .allocations - .lock() - .unwrap() - .get(&*ALLOCATION_ID_0) - .unwrap() - .as_active() - .unwrap() - .get_unaggregated_fees() - .value, - expected_unaggregated_fees - ); - - // Check that the unaggregated fees have *not* increased. - assert_eq!( - sender.inner.unaggregated_fees.lock().unwrap().value, - expected_unaggregated_fees - ); - } - - #[sqlx::test(migrations = "../migrations")] - async fn test_rav_requester_auto(pgpool: PgPool) { - // Start a TAP aggregator server. - let (handle, aggregator_endpoint) = run_server( - 0, - SIGNER.0.clone(), - vec![SIGNER.1].into_iter().collect(), - TAP_EIP712_DOMAIN_SEPARATOR.clone(), - 100 * 1024, - 100 * 1024, - 1, - ) - .await - .unwrap(); - - // Start a mock graphql server using wiremock - let mock_server = MockServer::start().await; - - // Mock result for TAP redeem txs for (allocation, sender) pair. - mock_server - .register( - Mock::given(method("POST")) - .and(body_string_contains("transactions")) - .respond_with( - ResponseTemplate::new(200) - .set_body_json(json!({ "data": { "transactions": []}})), - ), - ) - .await; - - // Create a sender_account. - let sender_account = create_sender_with_allocations( - pgpool.clone(), - "http://".to_owned() + &aggregator_endpoint.to_string(), - &mock_server.uri(), - ) - .await; - - // Add receipts to the database and call the `handle_new_receipt_notification` method - // correspondingly. - let mut total_value = 0; - let mut trigger_value = 0; - for i in 0..10 { - // These values should be enough to trigger a RAV request at i == 7 since we set the - // `rav_request_trigger_value` to 100. - let value = (i + 10) as u128; - - let receipt = - create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i + 1, value).await; - store_receipt(&pgpool, receipt.signed_receipt()) - .await - .unwrap(); - sender_account - .handle_new_receipt_notification(NewReceiptNotification { - allocation_id: *ALLOCATION_ID_0, - signer_address: SIGNER.1, - id: i, - timestamp_ns: i + 1, - value, - }) - .await; - - total_value += value; - if total_value >= 100 && trigger_value == 0 { - trigger_value = total_value; - } - } - - // Wait for the RAV requester to finish. - for _ in 0..100 { - if sender_account.inner.unaggregated_fees.lock().unwrap().value < trigger_value { - break; - } - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - } - - // Get the latest RAV from the database. - let latest_rav = sqlx::query!( - r#" - SELECT signature, allocation_id, timestamp_ns, value_aggregate - FROM scalar_tap_ravs - WHERE allocation_id = $1 AND sender_address = $2 - "#, - ALLOCATION_ID_0.encode_hex::(), - SENDER.1.encode_hex::() - ) - .fetch_optional(&pgpool) - .await - .unwrap() - .unwrap(); - - let latest_rav = EIP712SignedMessage { - message: ReceiptAggregateVoucher { - allocationId: Address::from_str(&latest_rav.allocation_id).unwrap(), - timestampNs: latest_rav.timestamp_ns.to_u64().unwrap(), - // Beware, BigDecimal::to_u128() actually uses to_u64() under the hood... - // So we're converting to BigInt to get a proper implementation of to_u128(). - valueAggregate: latest_rav - .value_aggregate - .to_bigint() - .map(|v| v.to_u128()) - .unwrap() - .unwrap(), - }, - signature: latest_rav.signature.as_slice().try_into().unwrap(), - }; - - // Check that the latest RAV value is correct. - assert!(latest_rav.message.valueAggregate >= trigger_value); - - // Check that the allocation's unaggregated fees value is reduced. - assert!( - sender_account - .inner - .allocations - .lock() - .unwrap() - .get(&*ALLOCATION_ID_0) - .unwrap() - .as_active() - .unwrap() - .get_unaggregated_fees() - .value - <= trigger_value - ); - - // Check that the sender's unaggregated fees value is reduced. - assert!(sender_account.inner.unaggregated_fees.lock().unwrap().value <= trigger_value); - - // Reset the total value and trigger value. - total_value = sender_account.inner.unaggregated_fees.lock().unwrap().value; - trigger_value = 0; - - // Add more receipts - for i in 10..20 { - let value = (i + 10) as u128; - - let receipt = - create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i + 1, i.into()).await; - store_receipt(&pgpool, receipt.signed_receipt()) - .await - .unwrap(); - - sender_account - .handle_new_receipt_notification(NewReceiptNotification { - allocation_id: *ALLOCATION_ID_0, - signer_address: SIGNER.1, - id: i, - timestamp_ns: i + 1, - value, - }) - .await; - - total_value += value; - if total_value >= 100 && trigger_value == 0 { - trigger_value = total_value; - } - } - - // Wait for the RAV requester to finish. - for _ in 0..100 { - if sender_account.inner.unaggregated_fees.lock().unwrap().value < trigger_value { - break; - } - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - } - - // Get the latest RAV from the database. - let latest_rav = sqlx::query!( - r#" - SELECT signature, allocation_id, timestamp_ns, value_aggregate - FROM scalar_tap_ravs - WHERE allocation_id = $1 AND sender_address = $2 - "#, - ALLOCATION_ID_0.encode_hex::(), - SENDER.1.encode_hex::() - ) - .fetch_optional(&pgpool) - .await - .unwrap() - .unwrap(); - - let latest_rav = EIP712SignedMessage { - message: ReceiptAggregateVoucher { - allocationId: Address::from_str(&latest_rav.allocation_id).unwrap(), - timestampNs: latest_rav.timestamp_ns.to_u64().unwrap(), - // Beware, BigDecimal::to_u128() actually uses to_u64() under the hood... - // So we're converting to BigInt to get a proper implementation of to_u128(). - valueAggregate: latest_rav - .value_aggregate - .to_bigint() - .map(|v| v.to_u128()) - .unwrap() - .unwrap(), - }, - signature: latest_rav.signature.as_slice().try_into().unwrap(), - }; - - // Check that the latest RAV value is correct. - - assert!(latest_rav.message.valueAggregate >= trigger_value); - - // Check that the allocation's unaggregated fees value is reduced. - assert!( - sender_account - .inner - .allocations - .lock() - .unwrap() - .get(&*ALLOCATION_ID_0) - .unwrap() - .as_active() - .unwrap() - .get_unaggregated_fees() - .value - <= trigger_value - ); - - // Check that the unaggregated fees value is reduced. - assert!(sender_account.inner.unaggregated_fees.lock().unwrap().value <= trigger_value); - - // Stop the TAP aggregator server. - handle.stop().unwrap(); - handle.stopped().await; - } - - #[sqlx::test(migrations = "../migrations")] - async fn test_sender_unaggregated_fees(pgpool: PgPool) { - // Create a sender_account. - let sender_account = Arc::new( - create_sender_with_allocations(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL).await, - ); - - // Closure that adds a number of receipts to an allocation. - let add_receipts = |allocation_id: Address, iterations: u64| { - let sender_account = sender_account.clone(); - - async move { - let mut total_value = 0; - for i in 0..iterations { - let value = (i + 10) as u128; - - let id = sender_account - .inner - .unaggregated_fees - .lock() - .unwrap() - .last_id - + 1; - - sender_account - .handle_new_receipt_notification(NewReceiptNotification { - allocation_id, - signer_address: SIGNER.1, - id, - timestamp_ns: i + 1, - value, - }) - .await; - - total_value += value; - } - - assert_eq!( - sender_account - .inner - .allocations - .lock() - .unwrap() - .get(&allocation_id) - .unwrap() - .as_active() - .unwrap() - .get_unaggregated_fees() - .value, - total_value - ); - - total_value - } - }; - - // Add receipts to the database for allocation_0 - let total_value_0 = add_receipts(*ALLOCATION_ID_0, 9).await; - - // Add receipts to the database for allocation_1 - let total_value_1 = add_receipts(*ALLOCATION_ID_1, 10).await; - - // Add receipts to the database for allocation_2 - let total_value_2 = add_receipts(*ALLOCATION_ID_2, 8).await; - - // Get the heaviest allocation. - let heaviest_allocation = sender_account.inner.get_heaviest_allocation().unwrap(); - - // Check that the heaviest allocation is correct. - assert_eq!(heaviest_allocation.get_allocation_id(), *ALLOCATION_ID_1); - - // Check that the sender's unaggregated fees value is correct. - assert_eq!( - sender_account.inner.unaggregated_fees.lock().unwrap().value, - total_value_0 + total_value_1 + total_value_2 - ); - } -} diff --git a/tap-agent/src/tap/sender_accounts_manager.rs b/tap-agent/src/tap/sender_accounts_manager.rs deleted file mode 100644 index 9800f898e..000000000 --- a/tap-agent/src/tap/sender_accounts_manager.rs +++ /dev/null @@ -1,526 +0,0 @@ -// Copyright 2023-, GraphOps and Semiotic Labs. -// SPDX-License-Identifier: Apache-2.0 - -use std::collections::HashSet; -use std::sync::Mutex as StdMutex; -use std::{collections::HashMap, str::FromStr, sync::Arc}; - -use alloy_sol_types::Eip712Domain; -use anyhow::anyhow; -use anyhow::Result; -use eventuals::{Eventual, EventualExt, PipeHandle}; -use indexer_common::escrow_accounts::EscrowAccounts; -use indexer_common::prelude::{Allocation, SubgraphClient}; -use serde::Deserialize; -use sqlx::{postgres::PgListener, PgPool}; -use thegraph::types::Address; -use tracing::{error, warn}; - -use crate::config; -use crate::tap::sender_account::SenderAccount; - -#[derive(Deserialize, Debug)] -pub struct NewReceiptNotification { - pub id: u64, - pub allocation_id: Address, - pub signer_address: Address, - pub timestamp_ns: u64, - pub value: u128, -} - -pub struct SenderAccountsManager { - _inner: Arc, - new_receipts_watcher_handle: tokio::task::JoinHandle<()>, - _eligible_allocations_senders_pipe: PipeHandle, -} - -/// Inner struct for SenderAccountsManager. This is used to store an Arc state for spawning async -/// tasks. -struct Inner { - config: &'static config::Cli, - pgpool: PgPool, - /// Map of sender_address to SenderAllocation. - sender_accounts: Arc>>>, - indexer_allocations: Eventual>, - escrow_accounts: Eventual, - escrow_subgraph: &'static SubgraphClient, - tap_eip712_domain_separator: Eip712Domain, - sender_aggregator_endpoints: HashMap, -} - -impl Inner { - async fn update_sender_accounts( - &self, - indexer_allocations: HashMap, - target_senders: HashSet
, - ) -> Result<()> { - let eligible_allocations: HashSet
= indexer_allocations.keys().copied().collect(); - let mut sender_accounts_copy = self.sender_accounts.lock().unwrap().clone(); - - // For all Senders that are not in the target_senders HashSet, set all their allocations to - // ineligible. That will trigger a finalization of all their receipts. - for (sender_id, sender_account) in sender_accounts_copy.iter() { - if !target_senders.contains(sender_id) { - sender_account.update_allocations(HashSet::new()).await; - } - } - - // Get or create SenderAccount instances for all currently eligible - // senders. - for sender_id in &target_senders { - let sender = - sender_accounts_copy - .entry(*sender_id) - .or_insert(Arc::new(SenderAccount::new( - self.config, - self.pgpool.clone(), - *sender_id, - self.escrow_accounts.clone(), - self.escrow_subgraph, - self.tap_eip712_domain_separator.clone(), - self.sender_aggregator_endpoints - .get(sender_id) - .ok_or_else(|| { - anyhow!( - "No sender_aggregator_endpoint found for sender {}", - sender_id - ) - })? - .clone(), - ))); - - // Update sender's allocations - sender - .update_allocations(eligible_allocations.clone()) - .await; - } - - // Replace the sender_accounts with the updated sender_accounts_copy - *self.sender_accounts.lock().unwrap() = sender_accounts_copy; - - // TODO: remove Sender instances that are finished. Ideally done in another async task? - - Ok(()) - } -} - -impl SenderAccountsManager { - pub async fn new( - config: &'static config::Cli, - pgpool: PgPool, - indexer_allocations: Eventual>, - escrow_accounts: Eventual, - escrow_subgraph: &'static SubgraphClient, - tap_eip712_domain_separator: Eip712Domain, - sender_aggregator_endpoints: HashMap, - ) -> Self { - let inner = Arc::new(Inner { - config, - pgpool, - sender_accounts: Arc::new(StdMutex::new(HashMap::new())), - indexer_allocations, - escrow_accounts, - escrow_subgraph, - tap_eip712_domain_separator, - sender_aggregator_endpoints, - }); - - // Listen to pg_notify events. We start it before updating the unaggregated_fees for all - // SenderAccount instances, so that we don't miss any receipts. PG will buffer the\ - // notifications until we start consuming them with `new_receipts_watcher`. - let mut pglistener = PgListener::connect_with(&inner.pgpool.clone()) - .await - .unwrap(); - pglistener - .listen("scalar_tap_receipt_notification") - .await - .expect( - "should be able to subscribe to Postgres Notify events on the channel \ - 'scalar_tap_receipt_notification'", - ); - - let escrow_accounts_snapshot = inner - .escrow_accounts - .value() - .await - .expect("Should get escrow accounts from Eventual"); - - // Gather all outstanding receipts and unfinalized RAVs from the database. - // Used to create SenderAccount instances for all senders that have unfinalized allocations - // and try to finalize them if they have become ineligible. - - // First we accumulate all allocations for each sender. This is because we may have more - // than one signer per sender in DB. - let mut unfinalized_sender_allocations_map: HashMap> = - HashMap::new(); - - let receipts_signer_allocations_in_db = sqlx::query!( - r#" - SELECT DISTINCT - signer_address, - ( - SELECT ARRAY - ( - SELECT DISTINCT allocation_id - FROM scalar_tap_receipts - WHERE signer_address = top.signer_address - ) - ) AS allocation_ids - FROM scalar_tap_receipts AS top - "# - ) - .fetch_all(&inner.pgpool) - .await - .expect("should be able to fetch pending receipts from the database"); - - for row in receipts_signer_allocations_in_db { - let allocation_ids = row - .allocation_ids - .expect("all receipts should have an allocation_id") - .iter() - .map(|allocation_id| { - Address::from_str(allocation_id) - .expect("allocation_id should be a valid address") - }) - .collect::>(); - let signer_id = Address::from_str(&row.signer_address) - .expect("signer_address should be a valid address"); - let sender_id = escrow_accounts_snapshot - .get_sender_for_signer(&signer_id) - .expect("should be able to get sender from signer"); - - // Accumulate allocations for the sender - unfinalized_sender_allocations_map - .entry(sender_id) - .or_default() - .extend(allocation_ids); - } - - let nonfinal_ravs_sender_allocations_in_db = sqlx::query!( - r#" - SELECT DISTINCT - sender_address, - ( - SELECT ARRAY - ( - SELECT DISTINCT allocation_id - FROM scalar_tap_ravs - WHERE sender_address = top.sender_address - ) - ) AS allocation_id - FROM scalar_tap_ravs AS top - "# - ) - .fetch_all(&inner.pgpool) - .await - .expect("should be able to fetch unfinalized RAVs from the database"); - - for row in nonfinal_ravs_sender_allocations_in_db { - let allocation_ids = row - .allocation_id - .expect("all RAVs should have an allocation_id") - .iter() - .map(|allocation_id| { - Address::from_str(allocation_id) - .expect("allocation_id should be a valid address") - }) - .collect::>(); - let sender_id = Address::from_str(&row.sender_address) - .expect("sender_address should be a valid address"); - - // Accumulate allocations for the sender - unfinalized_sender_allocations_map - .entry(sender_id) - .or_default() - .extend(allocation_ids); - } - - // Create SenderAccount instances for all senders that have unfinalized allocations and add - // the allocations to the SenderAccount instances. - let mut sender_accounts = HashMap::new(); - for (sender_id, allocation_ids) in unfinalized_sender_allocations_map { - let sender = sender_accounts - .entry(sender_id) - .or_insert(Arc::new(SenderAccount::new( - config, - inner.pgpool.clone(), - sender_id, - inner.escrow_accounts.clone(), - inner.escrow_subgraph, - inner.tap_eip712_domain_separator.clone(), - inner - .sender_aggregator_endpoints - .get(&sender_id) - .expect("should be able to get sender_aggregator_endpoint for sender") - .clone(), - ))); - - sender.update_allocations(allocation_ids).await; - - sender.recompute_unaggregated_fees().await; - } - // replace the sender_accounts with the updated sender_accounts - *inner.sender_accounts.lock().unwrap() = sender_accounts; - - // Update senders and allocations based on the current state of the network. - // It is important to do this after creating the Sender and SenderAllocation instances based - // on the receipts in the database, because now all ineligible allocation and/or sender that - // we created above will be set for receipt finalization. - inner - .update_sender_accounts( - inner - .indexer_allocations - .value() - .await - .expect("Should get indexer allocations from Eventual"), - escrow_accounts_snapshot.get_senders(), - ) - .await - .expect("Should be able to update_sender_accounts"); - - // Start the new_receipts_watcher task that will consume from the `pglistener` - let new_receipts_watcher_handle = tokio::spawn(Self::new_receipts_watcher( - pglistener, - inner.sender_accounts.clone(), - inner.escrow_accounts.clone(), - )); - - // Start the eligible_allocations_senders_pipe that watches for changes in eligible senders - // and allocations and updates the SenderAccount instances accordingly. - let inner_clone = inner.clone(); - let eligible_allocations_senders_pipe = eventuals::join(( - inner.indexer_allocations.clone(), - inner.escrow_accounts.clone(), - )) - .pipe_async(move |(indexer_allocations, escrow_accounts)| { - let inner = inner_clone.clone(); - async move { - inner - .update_sender_accounts(indexer_allocations, escrow_accounts.get_senders()) - .await - .unwrap_or_else(|e| { - error!("Error while updating sender_accounts: {:?}", e); - }); - } - }); - - Self { - _inner: inner, - new_receipts_watcher_handle, - _eligible_allocations_senders_pipe: eligible_allocations_senders_pipe, - } - } - - /// Continuously listens for new receipt notifications from Postgres and forwards them to the - /// corresponding SenderAccount. - async fn new_receipts_watcher( - mut pglistener: PgListener, - sender_accounts: Arc>>>, - escrow_accounts: Eventual, - ) { - loop { - // TODO: recover from errors or shutdown the whole program? - let pg_notification = pglistener.recv().await.expect( - "should be able to receive Postgres Notify events on the channel \ - 'scalar_tap_receipt_notification'", - ); - let new_receipt_notification: NewReceiptNotification = - serde_json::from_str(pg_notification.payload()).expect( - "should be able to deserialize the Postgres Notify event payload as a \ - NewReceiptNotification", - ); - - let sender_address = escrow_accounts - .value() - .await - .expect("should be able to get escrow accounts") - .get_sender_for_signer(&new_receipt_notification.signer_address); - - let sender_address = match sender_address { - Ok(sender_address) => sender_address, - Err(_) => { - error!( - "No sender address found for receipt signer address {}. \ - This should not happen.", - new_receipt_notification.signer_address - ); - // TODO: save the receipt in the failed receipts table? - continue; - } - }; - - let sender_account = sender_accounts - .lock() - .unwrap() - .get(&sender_address) - .cloned(); - - if let Some(sender_account) = sender_account { - sender_account - .handle_new_receipt_notification(new_receipt_notification) - .await; - } else { - warn!( - "No sender_allocation_manager found for sender_address {} to process new \ - receipt notification. This should not happen.", - sender_address - ); - } - } - } -} - -impl Drop for SenderAccountsManager { - fn drop(&mut self) { - // Abort the notification watcher on drop. Otherwise it may panic because the PgPool could - // get dropped before. (Observed in tests) - self.new_receipts_watcher_handle.abort(); - } -} - -#[cfg(test)] -mod tests { - - use std::vec; - - use ethereum_types::U256; - use indexer_common::{ - prelude::{AllocationStatus, SubgraphDeployment}, - subgraph_client::DeploymentDetails, - }; - use serde_json::json; - use thegraph::types::DeploymentId; - use wiremock::{ - matchers::{body_string_contains, method}, - Mock, MockServer, ResponseTemplate, - }; - - use crate::tap::test_utils::{INDEXER, SENDER, SIGNER, TAP_EIP712_DOMAIN_SEPARATOR}; - - use super::*; - - #[sqlx::test(migrations = "../migrations")] - async fn test_sender_account_creation_and_eol(pgpool: PgPool) { - let config = Box::leak(Box::new(config::Cli { - config: None, - ethereum: config::Ethereum { - indexer_address: INDEXER.1, - }, - tap: config::Tap { - rav_request_trigger_value: 100, - rav_request_timestamp_buffer_ms: 1, - ..Default::default() - }, - ..Default::default() - })); - - let (mut indexer_allocations_writer, indexer_allocations_eventual) = - Eventual::>::new(); - indexer_allocations_writer.write(HashMap::new()); - - let (mut escrow_accounts_writer, escrow_accounts_eventual) = - Eventual::::new(); - escrow_accounts_writer.write(EscrowAccounts::default()); - - // Mock escrow subgraph. - let mock_server = MockServer::start().await; - mock_server - .register( - Mock::given(method("POST")) - .and(body_string_contains("transactions")) - .respond_with( - ResponseTemplate::new(200) - .set_body_json(json!({ "data": { "transactions": []}})), - ), - ) - .await; - let escrow_subgraph = Box::leak(Box::new(SubgraphClient::new( - reqwest::Client::new(), - None, - DeploymentDetails::for_query_url(&mock_server.uri()).unwrap(), - ))); - - let sender_account = SenderAccountsManager::new( - config, - pgpool.clone(), - indexer_allocations_eventual, - escrow_accounts_eventual, - escrow_subgraph, - TAP_EIP712_DOMAIN_SEPARATOR.clone(), - HashMap::from([(SENDER.1, String::from("http://localhost:8000"))]), - ) - .await; - - let allocation_id = - Address::from_str("0xdd975e30aafebb143e54d215db8a3e8fd916a701").unwrap(); - - // Add an allocation to the indexer_allocations Eventual. - indexer_allocations_writer.write(HashMap::from([( - allocation_id, - Allocation { - id: allocation_id, - indexer: INDEXER.1, - allocated_tokens: U256::from_str("601726452999999979510903").unwrap(), - created_at_block_hash: - "0x99d3fbdc0105f7ccc0cd5bb287b82657fe92db4ea8fb58242dafb90b1c6e2adf".to_string(), - created_at_epoch: 953, - closed_at_epoch: None, - subgraph_deployment: SubgraphDeployment { - id: DeploymentId::from_str( - "0xcda7fa0405d6fd10721ed13d18823d24b535060d8ff661f862b26c23334f13bf" - ).unwrap(), - denied_at: Some(0), - }, - status: AllocationStatus::Null, - closed_at_epoch_start_block_hash: None, - previous_epoch_start_block_hash: None, - poi: None, - query_fee_rebates: None, - query_fees_collected: None, - }, - )])); - - // Add an escrow account to the escrow_accounts Eventual. - escrow_accounts_writer.write(EscrowAccounts::new( - HashMap::from([(SENDER.1, 1000.into())]), - HashMap::from([(SENDER.1, vec![SIGNER.1])]), - )); - - // Wait for the SenderAccount to be created. - tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; - - // Check that the SenderAccount was created. - assert!(sender_account - ._inner - .sender_accounts - .lock() - .unwrap() - .contains_key(&SENDER.1)); - - // Remove the escrow account from the escrow_accounts Eventual. - escrow_accounts_writer.write(EscrowAccounts::default()); - - // Wait a bit - tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; - - // Check that the Sender's allocation moved from active to ineligible. - assert!(sender_account - ._inner - .sender_accounts - .lock() - .unwrap() - .get(&SENDER.1) - .unwrap() - ._tests_get_allocations_active() - .is_empty()); - assert!(sender_account - ._inner - .sender_accounts - .lock() - .unwrap() - .get(&SENDER.1) - .unwrap() - ._tests_get_allocations_ineligible() - .contains_key(&allocation_id)); - } -} diff --git a/tap-agent/src/tap/sender_allocation.rs b/tap-agent/src/tap/sender_allocation.rs deleted file mode 100644 index e292ec70a..000000000 --- a/tap-agent/src/tap/sender_allocation.rs +++ /dev/null @@ -1,611 +0,0 @@ -// Copyright 2023-, GraphOps and Semiotic Labs. -// SPDX-License-Identifier: Apache-2.0 - -use std::{ - sync::{Arc, Mutex as StdMutex}, - time::Duration, -}; - -use alloy_primitives::hex::ToHex; -use alloy_sol_types::Eip712Domain; -use anyhow::{anyhow, ensure, Result}; -use bigdecimal::num_bigint::BigInt; -use eventuals::Eventual; -use indexer_common::{escrow_accounts::EscrowAccounts, prelude::SubgraphClient}; -use jsonrpsee::{core::client::ClientT, http_client::HttpClientBuilder, rpc_params}; -use sqlx::{types::BigDecimal, PgPool}; -use tap_aggregator::jsonrpsee_helpers::JsonRpcResponse; -use tap_core::{ - rav::{RAVRequest, ReceiptAggregateVoucher}, - receipt::{ - checks::{Check, Checks}, - Failed, ReceiptWithState, - }, - signed_message::EIP712SignedMessage, -}; -use thegraph::types::Address; -use tokio::sync::Mutex as TokioMutex; -use tracing::{error, warn}; - -use crate::{ - config::{self}, - tap::{signers_trimmed, unaggregated_receipts::UnaggregatedReceipts}, -}; - -use super::context::{checks::Signature, TapAgentContext}; -use super::{context::checks::AllocationId, escrow_adapter::EscrowAdapter}; - -type TapManager = tap_core::manager::Manager; - -/// Manages unaggregated fees and the TAP lifecyle for a specific (allocation, sender) pair. -pub struct SenderAllocation { - pgpool: PgPool, - tap_manager: TapManager, - allocation_id: Address, - sender: Address, - sender_aggregator_endpoint: String, - unaggregated_fees: Arc>, - config: &'static config::Cli, - escrow_accounts: Eventual, - rav_request_guard: TokioMutex<()>, - unaggregated_receipts_guard: TokioMutex<()>, - tap_eip712_domain_separator: Eip712Domain, -} - -impl SenderAllocation { - #[allow(clippy::too_many_arguments)] - pub async fn new( - config: &'static config::Cli, - pgpool: PgPool, - allocation_id: Address, - sender: Address, - escrow_accounts: Eventual, - escrow_subgraph: &'static SubgraphClient, - escrow_adapter: EscrowAdapter, - tap_eip712_domain_separator: Eip712Domain, - sender_aggregator_endpoint: String, - ) -> Self { - let required_checks: Vec> = vec![ - Arc::new(AllocationId::new( - sender, - allocation_id, - escrow_subgraph, - config, - )), - Arc::new(Signature::new( - tap_eip712_domain_separator.clone(), - escrow_accounts.clone(), - )), - ]; - let context = TapAgentContext::new( - pgpool.clone(), - allocation_id, - sender, - escrow_accounts.clone(), - escrow_adapter, - ); - let tap_manager = TapManager::new( - tap_eip712_domain_separator.clone(), - context, - Checks::new(required_checks), - ); - - let sender_allocation = Self { - pgpool, - tap_manager, - allocation_id, - sender, - sender_aggregator_endpoint, - unaggregated_fees: Arc::new(StdMutex::new(UnaggregatedReceipts::default())), - config, - escrow_accounts, - rav_request_guard: TokioMutex::new(()), - unaggregated_receipts_guard: TokioMutex::new(()), - tap_eip712_domain_separator, - }; - - sender_allocation - .update_unaggregated_fees() - .await - .map_err(|e| { - error!( - "Error while updating unaggregated fees for allocation {}: {}", - allocation_id, e - ) - }) - .ok(); - - sender_allocation - } - - /// Delete obsolete receipts in the DB w.r.t. the last RAV in DB, then update the tap manager - /// with the latest unaggregated fees from the database. - async fn update_unaggregated_fees(&self) -> Result<()> { - // Make sure to pause the handling of receipt notifications while we update the unaggregated - // fees. - let _guard = self.unaggregated_receipts_guard.lock().await; - - self.tap_manager.remove_obsolete_receipts().await?; - - let signers = signers_trimmed(&self.escrow_accounts, self.sender).await?; - - // TODO: Get `rav.timestamp_ns` from the TAP Manager's RAV storage adapter instead? - let res = sqlx::query!( - r#" - WITH rav AS ( - SELECT - timestamp_ns - FROM - scalar_tap_ravs - WHERE - allocation_id = $1 - AND sender_address = $2 - ) - SELECT - MAX(id), - SUM(value) - FROM - scalar_tap_receipts - WHERE - allocation_id = $1 - AND signer_address IN (SELECT unnest($3::text[])) - AND CASE WHEN ( - SELECT - timestamp_ns :: NUMERIC - FROM - rav - ) IS NOT NULL THEN timestamp_ns > ( - SELECT - timestamp_ns :: NUMERIC - FROM - rav - ) ELSE TRUE END - "#, - self.allocation_id.encode_hex::(), - self.sender.encode_hex::(), - &signers - ) - .fetch_one(&self.pgpool) - .await?; - - ensure!( - res.sum.is_none() == res.max.is_none(), - "Exactly one of SUM(value) and MAX(id) is null. This should not happen." - ); - - *self.unaggregated_fees.lock().unwrap() = UnaggregatedReceipts { - last_id: res.max.unwrap_or(0).try_into()?, - value: res - .sum - .unwrap_or(BigDecimal::from(0)) - .to_string() - .parse::()?, - }; - - // TODO: check if we need to run a RAV request here. - - Ok(()) - } - - /// Request a RAV from the sender's TAP aggregator. Only one RAV request will be running at a - /// time through the use of an internal guard. - pub async fn rav_requester_single(&self) -> Result<()> { - // Making extra sure that only one RAV request is running at a time. - let _guard = self.rav_request_guard.lock().await; - - let RAVRequest { - valid_receipts, - previous_rav, - invalid_receipts, - expected_rav, - } = self - .tap_manager - .create_rav_request( - self.config.tap.rav_request_timestamp_buffer_ms * 1_000_000, - // TODO: limit the number of receipts to aggregate per request. - None, - ) - .await - .map_err(|e| match e { - tap_core::Error::NoValidReceiptsForRAVRequest => anyhow!( - "It looks like there are no valid receipts for the RAV request.\ - This may happen if your `rav_request_trigger_value` is too low \ - and no receipts were found outside the `rav_request_timestamp_buffer_ms`.\ - You can fix this by increasing the `rav_request_trigger_value`." - ), - _ => e.into(), - })?; - if !invalid_receipts.is_empty() { - warn!( - "Found {} invalid receipts for allocation {} and sender {}.", - invalid_receipts.len(), - self.allocation_id, - self.sender - ); - - // Save invalid receipts to the database for logs. - // TODO: consider doing that in a spawned task? - Self::store_invalid_receipts(self, invalid_receipts.as_slice()).await?; - } - let client = HttpClientBuilder::default() - .request_timeout(Duration::from_secs( - self.config.tap.rav_request_timeout_secs, - )) - .build(&self.sender_aggregator_endpoint)?; - let response: JsonRpcResponse> = client - .request( - "aggregate_receipts", - rpc_params!( - "0.0", // TODO: Set the version in a smarter place. - valid_receipts, - previous_rav - ), - ) - .await?; - if let Some(warnings) = response.warnings { - warn!("Warnings from sender's TAP aggregator: {:?}", warnings); - } - match self - .tap_manager - .verify_and_store_rav(expected_rav.clone(), response.data.clone()) - .await - { - Ok(_) => {} - - // Adapter errors are local software errors. Shouldn't be a problem with the sender. - Err(tap_core::Error::AdapterError { source_error: e }) => { - anyhow::bail!("TAP Adapter error while storing RAV: {:?}", e) - } - - // The 3 errors below signal an invalid RAV, which should be about problems with the - // sender. The sender could be malicious. - Err( - e @ tap_core::Error::InvalidReceivedRAV { - expected_rav: _, - received_rav: _, - } - | e @ tap_core::Error::SignatureError(_) - | e @ tap_core::Error::InvalidRecoveredSigner { address: _ }, - ) => { - Self::store_failed_rav(self, &expected_rav, &response.data, &e.to_string()).await?; - anyhow::bail!("Invalid RAV, sender could be malicious: {:?}.", e); - } - - // All relevant errors should be handled above. If we get here, we forgot to handle - // an error case. - Err(e) => { - anyhow::bail!("Error while verifying and storing RAV: {:?}", e); - } - } - Self::update_unaggregated_fees(self).await?; - Ok(()) - } - - pub async fn mark_rav_last(&self) -> Result<()> { - let updated_rows = sqlx::query!( - r#" - UPDATE scalar_tap_ravs - SET last = true - WHERE allocation_id = $1 AND sender_address = $2 - "#, - self.allocation_id.encode_hex::(), - self.sender.encode_hex::(), - ) - .execute(&self.pgpool) - .await?; - if updated_rows.rows_affected() != 1 { - anyhow::bail!( - "Expected exactly one row to be updated in the latest RAVs table, \ - but {} were updated.", - updated_rows.rows_affected() - ); - }; - Ok(()) - } - - async fn store_invalid_receipts(&self, receipts: &[ReceiptWithState]) -> Result<()> { - for received_receipt in receipts.iter() { - let receipt = received_receipt.signed_receipt(); - let allocation_id = receipt.message.allocation_id; - let encoded_signature = receipt.signature.to_vec(); - - let receipt_signer = receipt - .recover_signer(&self.tap_eip712_domain_separator) - .map_err(|e| { - error!("Failed to recover receipt signer: {}", e); - anyhow!(e) - })?; - - sqlx::query!( - r#" - INSERT INTO scalar_tap_receipts_invalid ( - signer_address, - signature, - allocation_id, - timestamp_ns, - nonce, - value - ) - VALUES ($1, $2, $3, $4, $5, $6) - "#, - receipt_signer.encode_hex::(), - encoded_signature, - allocation_id.encode_hex::(), - BigDecimal::from(receipt.message.timestamp_ns), - BigDecimal::from(receipt.message.nonce), - BigDecimal::from(BigInt::from(receipt.message.value)), - ) - .execute(&self.pgpool) - .await - .map_err(|e| anyhow!("Failed to store failed receipt: {:?}", e))?; - } - - Ok(()) - } - - async fn store_failed_rav( - &self, - expected_rav: &ReceiptAggregateVoucher, - rav: &EIP712SignedMessage, - reason: &str, - ) -> Result<()> { - sqlx::query!( - r#" - INSERT INTO scalar_tap_rav_requests_failed ( - allocation_id, - sender_address, - expected_rav, - rav_response, - reason - ) - VALUES ($1, $2, $3, $4, $5) - "#, - self.allocation_id.encode_hex::(), - self.sender.encode_hex::(), - serde_json::to_value(expected_rav)?, - serde_json::to_value(rav)?, - reason - ) - .execute(&self.pgpool) - .await - .map_err(|e| anyhow!("Failed to store failed RAV: {:?}", e))?; - - Ok(()) - } - - /// Safe add the fees to the unaggregated fees value if the receipt_id is greater than the - /// last_id. If the addition would overflow u128, log an error and set the unaggregated fees - /// value to u128::MAX. - /// - /// Returns true if the fees were added, false otherwise. - pub async fn fees_add(&self, fees: u128, receipt_id: u64) -> bool { - // Make sure to pause the handling of receipt notifications while we update the unaggregated - // fees. - let _guard = self.unaggregated_receipts_guard.lock().await; - - let mut fees_added = false; - let mut unaggregated_fees = self.unaggregated_fees.lock().unwrap(); - - if receipt_id > unaggregated_fees.last_id { - *unaggregated_fees = UnaggregatedReceipts { - last_id: receipt_id, - value: unaggregated_fees - .value - .checked_add(fees) - .unwrap_or_else(|| { - // This should never happen, but if it does, we want to know about it. - error!( - "Overflow when adding receipt value {} to total unaggregated fees {} \ - for allocation {} and sender {}. Setting total unaggregated fees to \ - u128::MAX.", - fees, unaggregated_fees.value, self.allocation_id, self.sender - ); - u128::MAX - }), - }; - fees_added = true; - } - - fees_added - } - - pub fn get_unaggregated_fees(&self) -> UnaggregatedReceipts { - self.unaggregated_fees.lock().unwrap().clone() - } - - pub fn get_allocation_id(&self) -> Address { - self.allocation_id - } -} - -#[cfg(test)] -mod tests { - - use std::collections::HashMap; - - use indexer_common::subgraph_client::DeploymentDetails; - use serde_json::json; - use tap_aggregator::server::run_server; - - use wiremock::{ - matchers::{body_string_contains, method}, - Mock, MockServer, ResponseTemplate, - }; - - use super::*; - use crate::tap::test_utils::{ - create_rav, create_received_receipt, store_rav, store_receipt, ALLOCATION_ID_0, INDEXER, - SENDER, SIGNER, TAP_EIP712_DOMAIN_SEPARATOR, - }; - - const DUMMY_URL: &str = "http://localhost:1234"; - - async fn create_sender_allocation( - pgpool: PgPool, - sender_aggregator_endpoint: String, - escrow_subgraph_endpoint: &str, - ) -> SenderAllocation { - let config = Box::leak(Box::new(config::Cli { - config: None, - ethereum: config::Ethereum { - indexer_address: INDEXER.1, - }, - tap: config::Tap { - rav_request_trigger_value: 100, - rav_request_timestamp_buffer_ms: 1, - rav_request_timeout_secs: 5, - ..Default::default() - }, - ..Default::default() - })); - - let escrow_subgraph = Box::leak(Box::new(SubgraphClient::new( - reqwest::Client::new(), - None, - DeploymentDetails::for_query_url(escrow_subgraph_endpoint).unwrap(), - ))); - - let escrow_accounts_eventual = Eventual::from_value(EscrowAccounts::new( - HashMap::from([(SENDER.1, 1000.into())]), - HashMap::from([(SENDER.1, vec![SIGNER.1])]), - )); - - let escrow_adapter = EscrowAdapter::new(escrow_accounts_eventual.clone(), SENDER.1); - - SenderAllocation::new( - config, - pgpool.clone(), - *ALLOCATION_ID_0, - SENDER.1, - escrow_accounts_eventual, - escrow_subgraph, - escrow_adapter, - TAP_EIP712_DOMAIN_SEPARATOR.clone(), - sender_aggregator_endpoint, - ) - .await - } - - /// Test that the sender_allocation correctly updates the unaggregated fees from the - /// database when there is no RAV in the database. - /// - /// The sender_allocation should consider all receipts found for the allocation and - /// sender. - #[sqlx::test(migrations = "../migrations")] - async fn test_update_unaggregated_fees_no_rav(pgpool: PgPool) { - let sender_allocation = - create_sender_allocation(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL).await; - - // Add receipts to the database. - for i in 1..10 { - let receipt = - create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i, i.into()).await; - store_receipt(&pgpool, receipt.signed_receipt()) - .await - .unwrap(); - } - - // Let the sender_allocation update the unaggregated fees from the database. - sender_allocation.update_unaggregated_fees().await.unwrap(); - - // Check that the unaggregated fees are correct. - assert_eq!( - sender_allocation.unaggregated_fees.lock().unwrap().value, - 45u128 - ); - } - - /// Test that the sender_allocation correctly updates the unaggregated fees from the - /// database when there is a RAV in the database as well as receipts which timestamp are lesser - /// and greater than the RAV's timestamp. - /// - /// The sender_allocation should only consider receipts with a timestamp greater - /// than the RAV's timestamp. - #[sqlx::test(migrations = "../migrations")] - async fn test_update_unaggregated_fees_with_rav(pgpool: PgPool) { - let sender_allocation = - create_sender_allocation(pgpool.clone(), DUMMY_URL.to_string(), DUMMY_URL).await; - - // Add the RAV to the database. - // This RAV has timestamp 4. The sender_allocation should only consider receipts - // with a timestamp greater than 4. - let signed_rav = create_rav(*ALLOCATION_ID_0, SIGNER.0.clone(), 4, 10).await; - store_rav(&pgpool, signed_rav, SENDER.1).await.unwrap(); - - // Add receipts to the database. - for i in 1..10 { - let receipt = - create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i, i.into()).await; - store_receipt(&pgpool, receipt.signed_receipt()) - .await - .unwrap(); - } - - // Let the sender_allocation update the unaggregated fees from the database. - sender_allocation.update_unaggregated_fees().await.unwrap(); - - // Check that the unaggregated fees are correct. - assert_eq!( - sender_allocation.unaggregated_fees.lock().unwrap().value, - 35u128 - ); - } - - #[sqlx::test(migrations = "../migrations")] - async fn test_rav_requester_manual(pgpool: PgPool) { - // Start a TAP aggregator server. - let (handle, aggregator_endpoint) = run_server( - 0, - SIGNER.0.clone(), - vec![SIGNER.1].into_iter().collect(), - TAP_EIP712_DOMAIN_SEPARATOR.clone(), - 100 * 1024, - 100 * 1024, - 1, - ) - .await - .unwrap(); - - // Start a mock graphql server using wiremock - let mock_server = MockServer::start().await; - - // Mock result for TAP redeem txs for (allocation, sender) pair. - mock_server - .register( - Mock::given(method("POST")) - .and(body_string_contains("transactions")) - .respond_with( - ResponseTemplate::new(200) - .set_body_json(json!({ "data": { "transactions": []}})), - ), - ) - .await; - - // Create a sender_allocation. - let sender_allocation = create_sender_allocation( - pgpool.clone(), - "http://".to_owned() + &aggregator_endpoint.to_string(), - &mock_server.uri(), - ) - .await; - - // Add receipts to the database. - for i in 0..10 { - let receipt = - create_received_receipt(&ALLOCATION_ID_0, &SIGNER.0, i, i + 1, i.into()).await; - store_receipt(&pgpool, receipt.signed_receipt()) - .await - .unwrap(); - } - - // Let the sender_allocation update the unaggregated fees from the database. - sender_allocation.update_unaggregated_fees().await.unwrap(); - - // Trigger a RAV request manually. - sender_allocation.rav_requester_single().await.unwrap(); - - // Stop the TAP aggregator server. - handle.stop().unwrap(); - handle.stopped().await; - } -} diff --git a/tap-agent/src/tap/test_utils.rs b/tap-agent/src/tap/test_utils.rs index a93e51df7..0a1f06c09 100644 --- a/tap-agent/src/tap/test_utils.rs +++ b/tap-agent/src/tap/test_utils.rs @@ -4,12 +4,14 @@ use std::str::FromStr; use alloy_primitives::hex::ToHex; -use alloy_sol_types::{eip712_domain, Eip712Domain}; -use anyhow::Result; use bigdecimal::num_bigint::BigInt; + +use sqlx::types::BigDecimal; + +use alloy_sol_types::{eip712_domain, Eip712Domain}; use ethers_signers::{coins_bip39::English, LocalWallet, MnemonicBuilder, Signer}; use lazy_static::lazy_static; -use sqlx::{types::BigDecimal, PgPool}; +use sqlx::PgPool; use tap_core::{ rav::{ReceiptAggregateVoucher, SignedRAV}, receipt::{Checking, Receipt, ReceiptWithState, SignedReceipt}, @@ -22,12 +24,8 @@ lazy_static! { Address::from_str("0xabababababababababababababababababababab").unwrap(); pub static ref ALLOCATION_ID_1: Address = Address::from_str("0xbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbcbc").unwrap(); - pub static ref ALLOCATION_ID_2: Address = - Address::from_str("0xcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd").unwrap(); - pub static ref ALLOCATION_ID_IRRELEVANT: Address = - Address::from_str("0xbcdebcdebcdebcdebcdebcdebcdebcdebcdebcde").unwrap(); pub static ref SENDER: (LocalWallet, Address) = wallet(0); - pub static ref SENDER_IRRELEVANT: (LocalWallet, Address) = wallet(1); + pub static ref SENDER_2: (LocalWallet, Address) = wallet(1); pub static ref SIGNER: (LocalWallet, Address) = wallet(2); pub static ref INDEXER: (LocalWallet, Address) = wallet(3); pub static ref TAP_EIP712_DOMAIN_SEPARATOR: Eip712Domain = eip712_domain! { @@ -38,21 +36,28 @@ lazy_static! { }; } -/// Fixture to generate a wallet and address -pub fn wallet(index: u32) -> (LocalWallet, Address) { - let wallet: LocalWallet = MnemonicBuilder::::default() - .phrase("abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon about") - .index(index) - .unwrap() - .build() - .unwrap(); - let address = wallet.address(); - (wallet, Address::from_slice(address.as_bytes())) +/// Fixture to generate a RAV using the wallet from `keys()` +pub fn create_rav( + allocation_id: Address, + signer_wallet: LocalWallet, + timestamp_ns: u64, + value_aggregate: u128, +) -> SignedRAV { + EIP712SignedMessage::new( + &TAP_EIP712_DOMAIN_SEPARATOR, + ReceiptAggregateVoucher { + allocationId: allocation_id, + timestampNs: timestamp_ns, + valueAggregate: value_aggregate, + }, + &signer_wallet, + ) + .unwrap() } /// Fixture to generate a signed receipt using the wallet from `keys()` and the /// given `query_id` and `value` -pub async fn create_received_receipt( +pub fn create_received_receipt( allocation_id: &Address, signer_wallet: &LocalWallet, nonce: u64, @@ -73,26 +78,7 @@ pub async fn create_received_receipt( ReceiptWithState::new(receipt) } -/// Fixture to generate a RAV using the wallet from `keys()` -pub async fn create_rav( - allocation_id: Address, - signer_wallet: LocalWallet, - timestamp_ns: u64, - value_aggregate: u128, -) -> SignedRAV { - EIP712SignedMessage::new( - &TAP_EIP712_DOMAIN_SEPARATOR, - ReceiptAggregateVoucher { - allocationId: allocation_id, - timestampNs: timestamp_ns, - valueAggregate: value_aggregate, - }, - &signer_wallet, - ) - .unwrap() -} - -pub async fn store_receipt(pgpool: &PgPool, signed_receipt: &SignedReceipt) -> Result { +pub async fn store_receipt(pgpool: &PgPool, signed_receipt: &SignedReceipt) -> anyhow::Result { let encoded_signature = signed_receipt.signature.to_vec(); let record = sqlx::query!( @@ -119,7 +105,23 @@ pub async fn store_receipt(pgpool: &PgPool, signed_receipt: &SignedReceipt) -> R Ok(id) } -pub async fn store_rav(pgpool: &PgPool, signed_rav: SignedRAV, sender: Address) -> Result<()> { +/// Fixture to generate a wallet and address +pub fn wallet(index: u32) -> (LocalWallet, Address) { + let wallet: LocalWallet = MnemonicBuilder::::default() + .phrase("abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon about") + .index(index) + .unwrap() + .build() + .unwrap(); + let address = wallet.address(); + (wallet, Address::from_slice(address.as_bytes())) +} + +pub async fn store_rav( + pgpool: &PgPool, + signed_rav: SignedRAV, + sender: Address, +) -> anyhow::Result<()> { let signature_bytes = signed_rav.signature.to_vec(); let _fut = sqlx::query!(