diff --git a/Cargo.lock b/Cargo.lock index 991ee2d2b79..0757d49ed11 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5458,6 +5458,7 @@ dependencies = [ "num_cpus", "proptest", "proptest-derive", + "rand 0.8.5", "serde", "serde_json", "thiserror", diff --git a/zebra-chain/src/transaction/unmined.rs b/zebra-chain/src/transaction/unmined.rs index 68361e20b6d..44f8161a614 100644 --- a/zebra-chain/src/transaction/unmined.rs +++ b/zebra-chain/src/transaction/unmined.rs @@ -290,7 +290,7 @@ impl From<&Arc> for UnminedTx { /// A verified unmined transaction, and the corresponding transaction fee. /// /// This transaction has been fully verified, in the context of the mempool. -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, PartialEq)] #[cfg_attr(any(test, feature = "proptest-impl"), derive(Arbitrary))] pub struct VerifiedUnminedTx { /// The unmined transaction. @@ -302,6 +302,14 @@ pub struct VerifiedUnminedTx { /// The number of legacy signature operations in this transaction's /// transparent inputs and outputs. pub legacy_sigop_count: u64, + + /// The block production fee weight for `transaction`, as defined by [ZIP-317]. + /// + /// This is not consensus-critical, so we use `f32` for efficient calculations + /// when the mempool holds a large number of transactions. + /// + /// [ZIP-317]: https://zips.z.cash/zip-0317#block-production + pub block_production_fee_weight: f32, } impl fmt::Display for VerifiedUnminedTx { @@ -315,19 +323,31 @@ impl fmt::Display for VerifiedUnminedTx { } impl VerifiedUnminedTx { - /// Create a new verified unmined transaction from a transaction, its fee and the legacy sigop count. + /// Create a new verified unmined transaction from an unmined transaction, + /// its miner fee, and its legacy sigop count. pub fn new( transaction: UnminedTx, miner_fee: Amount, legacy_sigop_count: u64, ) -> Self { + let block_production_fee_weight = + zip317::block_production_fee_weight(&transaction, miner_fee); + Self { transaction, miner_fee, legacy_sigop_count, + block_production_fee_weight, } } + /// Returns `true` if the transaction pays at least the [ZIP-317] conventional fee. + /// + /// [ZIP-317]: https://zips.z.cash/zip-0317#mempool-size-limiting + pub fn pays_conventional_fee(&self) -> bool { + self.miner_fee >= self.transaction.conventional_fee + } + /// The cost in bytes of the transaction, as defined in [ZIP-401]. /// /// A reflection of the work done by the network in processing them (proof @@ -365,7 +385,7 @@ impl VerifiedUnminedTx { pub fn eviction_weight(&self) -> u64 { let mut cost = self.cost(); - if self.miner_fee < self.transaction.conventional_fee { + if !self.pays_conventional_fee() { cost += MEMPOOL_TRANSACTION_LOW_FEE_PENALTY } diff --git a/zebra-chain/src/transaction/unmined/zip317.rs b/zebra-chain/src/transaction/unmined/zip317.rs index 74309104a8e..79ea0360d08 100644 --- a/zebra-chain/src/transaction/unmined/zip317.rs +++ b/zebra-chain/src/transaction/unmined/zip317.rs @@ -1,18 +1,15 @@ -//! The [ZIP-317 conventional fee calculation](https://zips.z.cash/zip-0317#fee-calculation) -//! for [UnminedTx]s. +//! An implementation of the [ZIP-317] fee calculations for [UnminedTx]s: +//! - [conventional fee](https://zips.z.cash/zip-0317#fee-calculation) +//! - [block production transaction weight](https://zips.z.cash/zip-0317#block-production) use std::cmp::max; use crate::{ amount::{Amount, NonNegative}, serialization::ZcashSerialize, - transaction::Transaction, + transaction::{Transaction, UnminedTx}, }; -// For doc links -#[allow(unused_imports)] -use crate::transaction::UnminedTx; - /// The marginal fee for the ZIP-317 fee calculation, in zatoshis per logical action. // // TODO: allow Amount in constants @@ -27,6 +24,26 @@ const P2PKH_STANDARD_INPUT_SIZE: usize = 150; /// The standard size of p2pkh outputs for the ZIP-317 fee calculation, in bytes. const P2PKH_STANDARD_OUTPUT_SIZE: usize = 34; +/// The recommended weight cap for ZIP-317 block production. +const MAX_BLOCK_PRODUCTION_WEIGHT: f32 = 4.0; + +/// Zebra's custom minimum weight for ZIP-317 block production, +/// based on half the [ZIP-203] recommended transaction expiry height of 40 blocks. +/// +/// This ensures all transactions have a non-zero probability of being mined, +/// which simplifies our implementation. +/// +/// If blocks are full, this makes it likely that very low fee transactions +/// will be mined: +/// - after approximately 20 blocks delay, +/// - but before they expire. +/// +/// Note: Small transactions that pay the legacy ZIP-313 conventional fee have twice this weight. +/// If blocks are full, they will be mined after approximately 10 blocks delay. +/// +/// [ZIP-203]: https://zips.z.cash/zip-0203#changes-for-blossom> +const MIN_BLOCK_PRODUCTION_WEIGHT: f32 = 1.0 / 20.0; + /// Returns the conventional fee for `transaction`, as defined by [ZIP-317]. /// /// [ZIP-317]: https://zips.z.cash/zip-0317#fee-calculation @@ -72,6 +89,18 @@ pub fn conventional_fee(transaction: &Transaction) -> Amount { conventional_fee.expect("conventional fee is positive and limited by serialized size limit") } +/// Returns the block production fee weight for `transaction`, as defined by [ZIP-317]. +/// +/// [ZIP-317]: https://zips.z.cash/zip-0317#block-production +pub fn block_production_fee_weight(transaction: &UnminedTx, miner_fee: Amount) -> f32 { + let miner_fee = i64::from(miner_fee) as f32; + let conventional_fee = i64::from(transaction.conventional_fee) as f32; + + let uncapped_weight = miner_fee / conventional_fee; + + uncapped_weight.clamp(MIN_BLOCK_PRODUCTION_WEIGHT, MAX_BLOCK_PRODUCTION_WEIGHT) +} + /// Divide `quotient` by `divisor`, rounding the result up to the nearest integer. /// /// # Correctness diff --git a/zebra-consensus/src/transaction.rs b/zebra-consensus/src/transaction.rs index 55e6539ca91..c77028098f7 100644 --- a/zebra-consensus/src/transaction.rs +++ b/zebra-consensus/src/transaction.rs @@ -117,7 +117,7 @@ pub enum Request { /// The response type for the transaction verifier service. /// Responses identify the transaction that was verified. -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub enum Response { /// A response to a block transaction verification request. Block { diff --git a/zebra-rpc/Cargo.toml b/zebra-rpc/Cargo.toml index f4eff81e93a..1c6a55fbcc1 100644 --- a/zebra-rpc/Cargo.toml +++ b/zebra-rpc/Cargo.toml @@ -14,6 +14,7 @@ default = [] # Experimental mining RPC support getblocktemplate-rpcs = [ + "rand", "zebra-consensus/getblocktemplate-rpcs", "zebra-state/getblocktemplate-rpcs", "zebra-node-services/getblocktemplate-rpcs", @@ -54,6 +55,10 @@ tracing-futures = "0.2.5" hex = { version = "0.4.3", features = ["serde"] } serde = { version = "1.0.147", features = ["serde_derive"] } +# Experimental feature getblocktemplate-rpcs +rand = { version = "0.8.5", package = "rand", optional = true } + +# Test-only feature proptest-impl proptest = { version = "0.10.1", optional = true } proptest-derive = { version = "0.3.0", optional = true } diff --git a/zebra-rpc/src/methods/get_block_template_rpcs.rs b/zebra-rpc/src/methods/get_block_template_rpcs.rs index 9b8a1bbb08e..bb5b639f10a 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs.rs @@ -38,7 +38,9 @@ use crate::methods::{ pub mod config; pub mod constants; + pub(crate) mod types; +pub(crate) mod zip317; /// The max estimated distance to the chain tip for the getblocktemplate method // Set to 30 in case the local time is a little ahead. @@ -310,7 +312,7 @@ where }); } - let mempool_txs = select_mempool_transactions(mempool).await?; + let mempool_txs = zip317::select_mempool_transactions(mempool).await?; let miner_fee = miner_fee(&mempool_txs); @@ -448,37 +450,6 @@ where // get_block_template support methods -/// Returns selected transactions in the `mempool`, or an error if the mempool has failed. -/// -/// TODO: select transactions according to ZIP-317 (#5473) -pub async fn select_mempool_transactions( - mempool: Mempool, -) -> Result> -where - Mempool: Service< - mempool::Request, - Response = mempool::Response, - Error = zebra_node_services::BoxError, - > + 'static, - Mempool::Future: Send, -{ - let response = mempool - .oneshot(mempool::Request::FullTransactions) - .await - .map_err(|error| Error { - code: ErrorCode::ServerError(0), - message: error.to_string(), - data: None, - })?; - - if let mempool::Response::FullTransactions(transactions) = response { - // TODO: select transactions according to ZIP-317 (#5473) - Ok(transactions) - } else { - unreachable!("unmatched response to a mempool::FullTransactions request"); - } -} - /// Returns the total miner fee for `mempool_txs`. pub fn miner_fee(mempool_txs: &[VerifiedUnminedTx]) -> Amount { let miner_fee: amount::Result> = diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs b/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs new file mode 100644 index 00000000000..6f1a5114839 --- /dev/null +++ b/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs @@ -0,0 +1,186 @@ +//! The [ZIP-317 block production algorithm](https://zips.z.cash/zip-0317#block-production). +//! +//! This is recommended algorithm, so these calculations are not consensus-critical, +//! or standardised across node implementations: +//! > it is sufficient to use floating point arithmetic to calculate the argument to `floor` +//! > when computing `size_target`, since there is no consensus requirement for this to be +//! > exactly the same between implementations. + +use jsonrpc_core::{Error, ErrorCode, Result}; +use rand::{ + distributions::{Distribution, WeightedIndex}, + prelude::thread_rng, +}; +use tower::{Service, ServiceExt}; + +use zebra_chain::{block::MAX_BLOCK_BYTES, transaction::VerifiedUnminedTx}; +use zebra_consensus::MAX_BLOCK_SIGOPS; +use zebra_node_services::mempool; + +/// Selects mempool transactions for block production according to [ZIP-317]. +/// +/// Returns selected transactions from the `mempool`, or an error if the mempool has failed. +/// +/// [ZIP-317]: https://zips.z.cash/zip-0317#block-production +pub async fn select_mempool_transactions( + mempool: Mempool, +) -> Result> +where + Mempool: Service< + mempool::Request, + Response = mempool::Response, + Error = zebra_node_services::BoxError, + > + 'static, + Mempool::Future: Send, +{ + let mempool_transactions = fetch_mempool_transactions(mempool).await?; + + // Setup the transaction lists. + let (conventional_fee_txs, low_fee_txs): (Vec<_>, Vec<_>) = mempool_transactions + .into_iter() + .partition(VerifiedUnminedTx::pays_conventional_fee); + + // Set up limit tracking + let mut selected_txs = Vec::new(); + let mut remaining_block_sigops = MAX_BLOCK_SIGOPS; + let mut remaining_block_bytes: usize = MAX_BLOCK_BYTES.try_into().expect("fits in memory"); + + if let Some((conventional_fee_tx_weights, _total_weight)) = + setup_fee_weighted_index(&conventional_fee_txs) + { + let mut conventional_fee_tx_weights = Some(conventional_fee_tx_weights); + + // > Repeat while there is any mempool transaction that: + // > - pays at least the conventional fee, + // > - is within the block sigop limit, and + // > - fits in the block... + while let Some(tx_weights) = conventional_fee_tx_weights { + // > Pick one of those transactions at random with probability in direct proportion + // > to its weight, and add it to the block. + let (tx_weights, candidate_tx) = + choose_transaction_weighted_random(&conventional_fee_txs, tx_weights); + conventional_fee_tx_weights = tx_weights; + + if candidate_tx.legacy_sigop_count <= remaining_block_sigops + && candidate_tx.transaction.size <= remaining_block_bytes + { + selected_txs.push(candidate_tx.clone()); + + remaining_block_sigops -= candidate_tx.legacy_sigop_count; + remaining_block_bytes -= candidate_tx.transaction.size; + } + } + } + + // > Let `N` be the number of remaining transactions with `tx.weight < 1`. + // > Calculate their sum of weights. + if let Some((low_fee_tx_weights, remaining_weight)) = setup_fee_weighted_index(&low_fee_txs) { + let low_fee_tx_count = low_fee_txs.len() as f32; + + // > Calculate `size_target = ...` + // + // We track the remaining bytes within our scaled quota, + // so there is no need to actually calculate `size_target` or `size_of_block_so_far`. + let average_remaining_weight = remaining_weight / low_fee_tx_count; + + let remaining_block_bytes = + remaining_block_bytes as f32 * average_remaining_weight.min(1.0); + let mut remaining_block_bytes = remaining_block_bytes as usize; + + let mut low_fee_tx_weights = Some(low_fee_tx_weights); + + while let Some(tx_weights) = low_fee_tx_weights { + // > Pick a transaction with probability in direct proportion to its weight... + let (tx_weights, candidate_tx) = + choose_transaction_weighted_random(&low_fee_txs, tx_weights); + low_fee_tx_weights = tx_weights; + + // > and add it to the block. If that transaction would exceed the `size_target` + // > or the block sigop limit, stop without adding it. + if candidate_tx.legacy_sigop_count > remaining_block_sigops + || candidate_tx.transaction.size > remaining_block_bytes + { + // We've exceeded the scaled quota size limit, or the absolute sigop limit + break; + } + + selected_txs.push(candidate_tx.clone()); + + remaining_block_sigops -= candidate_tx.legacy_sigop_count; + remaining_block_bytes -= candidate_tx.transaction.size; + } + } + + Ok(selected_txs) +} + +/// Fetch the transactions that are currently in `mempool`. +async fn fetch_mempool_transactions(mempool: Mempool) -> Result> +where + Mempool: Service< + mempool::Request, + Response = mempool::Response, + Error = zebra_node_services::BoxError, + > + 'static, + Mempool::Future: Send, +{ + let response = mempool + .oneshot(mempool::Request::FullTransactions) + .await + .map_err(|error| Error { + code: ErrorCode::ServerError(0), + message: error.to_string(), + data: None, + })?; + + if let mempool::Response::FullTransactions(transactions) = response { + Ok(transactions) + } else { + unreachable!("unmatched response to a mempool::FullTransactions request") + } +} + +/// Returns a fee-weighted index and the total weight of `transactions`. +/// +/// Returns `None` if there are no transactions, or if the weights are invalid. +fn setup_fee_weighted_index( + transactions: &[VerifiedUnminedTx], +) -> Option<(WeightedIndex, f32)> { + if transactions.is_empty() { + return None; + } + + let tx_weights: Vec = transactions + .iter() + .map(|tx| tx.block_production_fee_weight) + .collect(); + let total_tx_weight: f32 = tx_weights.iter().sum(); + + // Setup the transaction weights. + let tx_weights = WeightedIndex::new(tx_weights).ok()?; + + Some((tx_weights, total_tx_weight)) +} + +/// Choose a transaction from `transactions`, using the previously set up `weighted_index`. +/// +/// If some transactions have not yet been chosen, returns the weighted index and the transaction. +/// Otherwise, just returns the transaction. +fn choose_transaction_weighted_random( + transactions: &[VerifiedUnminedTx], + mut weighted_index: WeightedIndex, +) -> (Option>, VerifiedUnminedTx) { + let candidate_position = weighted_index.sample(&mut thread_rng()); + let candidate_tx = transactions[candidate_position].clone(); + + // Only pick each transaction once, by setting picked transaction weights to zero + if weighted_index + .update_weights(&[(candidate_position, &0.0)]) + .is_err() + { + // All weights are zero, so each transaction has either been selected or rejected + (None, candidate_tx) + } else { + (Some(weighted_index), candidate_tx) + } +} diff --git a/zebrad/src/components/mempool/storage/verified_set.rs b/zebrad/src/components/mempool/storage/verified_set.rs index 548fc43fb89..c3abb31ba39 100644 --- a/zebrad/src/components/mempool/storage/verified_set.rs +++ b/zebrad/src/components/mempool/storage/verified_set.rs @@ -161,7 +161,7 @@ impl VerifiedSet { .collect(); let dist = WeightedIndex::new(weights) - .expect("there is at least one weight and all weights are valid"); + .expect("there is at least one weight, all weights are non-negative, and the total is positive"); Some(self.remove(dist.sample(&mut thread_rng()))) }