change(rpc): Populate blockcommitmenthash and defaultroot fields …

…in the getblocktemplate RPC (#5751)

* populate `blockcommitmenthash` and `defaultroot` missing fields

* remove assertion line manually from snaps

* fix some imports and docs

* fix some docs

* add a consistency check

* Rename a constant to FINALIZED_STATE_QUERY_RETRIES and use it everywhere

* Move tip query inside retry, split tip into tip_height and tip_hash

* Return retry failures rather than panicking

* Query relevant chain inside the retry

* Check the entire state for consistency, not just the finalized tip

Co-authored-by: teor <teor@riseup.net>

zebra-rpc/src/methods/get_block_template_rpcs.rs

@@ -9,11 +9,10 @@ use tower::{buffer::Buffer, Service, ServiceExt};
 
 use zebra_chain::{
     amount::{self, Amount, NonNegative},
-    block::Height,
     block::{
         self,
         merkle::{self, AuthDataRoot},
-        Block, MAX_BLOCK_BYTES, ZCASH_BLOCK_VERSION,
+        Block, ChainHistoryBlockTxAuthCommitmentHash, Height, MAX_BLOCK_BYTES, ZCASH_BLOCK_VERSION,
     },
     chain_sync_status::ChainSyncStatus,
     chain_tip::ChainTip,
@@ -338,7 +337,7 @@ where
 
             // The tip estimate may not be the same as the one coming from the state
             // but this is ok for an estimate
-            let (estimated_distance_to_chain_tip, tip_height) = latest_chain_tip
+            let (estimated_distance_to_chain_tip, estimated_tip_height) = latest_chain_tip
                 .estimate_distance_to_network_chain_tip(network)
                 .ok_or_else(|| Error {
                     code: ErrorCode::ServerError(0),
@@ -349,7 +348,7 @@ where
             if !sync_status.is_close_to_tip() || estimated_distance_to_chain_tip > MAX_ESTIMATED_DISTANCE_TO_NETWORK_CHAIN_TIP {
                 tracing::info!(
                     estimated_distance_to_chain_tip,
-                    ?tip_height,
+                    ?estimated_tip_height,
                     "Zebra has not synced to the chain tip"
                 );
 
@@ -377,15 +376,12 @@ where
                             })?;
 
             let chain_info = match response {
-                ReadResponse::ChainInfo(Some(chain_info)) => chain_info,
+                ReadResponse::ChainInfo(chain_info) => chain_info,
                 _ => unreachable!("we should always have enough state data here to get a `GetBlockTemplateChainInfo`"),
             };
 
             // Get the tip data from the state call
-            let tip_height = chain_info.tip.0;
-            let tip_hash = chain_info.tip.1;
-
-            let block_height = (tip_height + 1).expect("tip is far below Height::MAX");
+            let block_height = (chain_info.tip_height + 1).expect("tip is far below Height::MAX");
 
             let outputs =
                 standard_coinbase_outputs(network, block_height, miner_address, miner_fee);
@@ -394,6 +390,16 @@ where
             let (merkle_root, auth_data_root) =
                 calculate_transaction_roots(&coinbase_tx, &mempool_txs);
 
+            let history_tree = chain_info.history_tree;
+            // TODO: move expensive cryptography to a rayon thread?
+            let chain_history_root = history_tree.hash().expect("history tree can't be empty");
+
+            // TODO: move expensive cryptography to a rayon thread?
+            let block_commitments_hash = ChainHistoryBlockTxAuthCommitmentHash::from_commitments(
+                &chain_history_root,
+                &auth_data_root,
+            );
+
             // Convert into TransactionTemplates
             let mempool_txs = mempool_txs.iter().map(Into::into).collect();
 
@@ -404,15 +410,15 @@ where
 
                 version: ZCASH_BLOCK_VERSION,
 
-                previous_block_hash: GetBlockHash(tip_hash),
-                block_commitments_hash: [0; 32].into(),
-                light_client_root_hash: [0; 32].into(),
-                final_sapling_root_hash: [0; 32].into(),
+                previous_block_hash: GetBlockHash(chain_info.tip_hash),
+                block_commitments_hash,
+                light_client_root_hash: block_commitments_hash,
+                final_sapling_root_hash: block_commitments_hash,
                 default_roots: DefaultRoots {
                     merkle_root,
-                    chain_history_root: [0; 32].into(),
+                    chain_history_root,
                     auth_data_root,
-                    block_commitments_hash: [0; 32].into(),
+                    block_commitments_hash,
                 },
 
                 transactions: mempool_txs,

zebra-rpc/src/methods/tests.rs

@@ -2,4 +2,6 @@
 
 mod prop;
 mod snapshot;
+#[cfg(feature = "getblocktemplate-rpcs")]
+pub mod utils;
 mod vectors;

zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs

@@ -31,6 +31,7 @@ use crate::methods::{
         self,
         types::{get_block_template::GetBlockTemplate, hex_data::HexData, submit_block},
     },
+    tests::utils::fake_history_tree,
     GetBlockHash, GetBlockTemplateRpc, GetBlockTemplateRpcImpl,
 };
 
@@ -154,13 +155,15 @@ pub async fn test_responses<State, ReadState>(
             .clone()
             .expect_request_that(|req| matches!(req, ReadRequest::ChainInfo))
             .await
-            .respond(ReadResponse::ChainInfo(Some(GetBlockTemplateChainInfo {
+            .respond(ReadResponse::ChainInfo(GetBlockTemplateChainInfo {
                 expected_difficulty: CompactDifficulty::from(ExpandedDifficulty::from(U256::one())),
-                tip: (fake_tip_height, fake_tip_hash),
+                tip_height: fake_tip_height,
+                tip_hash: fake_tip_hash,
                 cur_time: fake_cur_time,
                 min_time: fake_min_time,
                 max_time: fake_max_time,
-            })));
+                history_tree: fake_history_tree(network),
+            }));
     });
 
     let get_block_template = tokio::spawn(get_block_template_rpc.get_block_template(None));

zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template@mainnet_10.snap

@@ -6,14 +6,14 @@ expression: block_template
   "capabilities": [],
   "version": 4,
   "previousblockhash": "0000000000d723156d9b65ffcf4984da7a19675ed7e2f06d9e5d5188af087bf8",
-  "blockcommitmentshash": "0000000000000000000000000000000000000000000000000000000000000000",
-  "lightclientroothash": "0000000000000000000000000000000000000000000000000000000000000000",
-  "finalsaplingroothash": "0000000000000000000000000000000000000000000000000000000000000000",
+  "blockcommitmentshash": "fe03d8236b0835c758f59d279230ebaee2128754413103b9edb17c07451c2c82",
+  "lightclientroothash": "fe03d8236b0835c758f59d279230ebaee2128754413103b9edb17c07451c2c82",
+  "finalsaplingroothash": "fe03d8236b0835c758f59d279230ebaee2128754413103b9edb17c07451c2c82",
   "defaultroots": {
     "merkleroot": "6b370584714ab567c9c014ce72d325ab6c5927e181ac891acb35e6d4b6cc19a1",
-    "chainhistoryroot": "0000000000000000000000000000000000000000000000000000000000000000",
+    "chainhistoryroot": "94470fa66ebd1a5fdb109a5aa3f3204f14de3a42135e71aa7f4c44055847e0b5",
     "authdataroot": "0dbb78de9fdcd494307971e36dd049fc82d0ee9ee53aec8fd2a54dc0e426289b",
-    "blockcommitmentshash": "0000000000000000000000000000000000000000000000000000000000000000"
+    "blockcommitmentshash": "fe03d8236b0835c758f59d279230ebaee2128754413103b9edb17c07451c2c82"
   },
   "transactions": [],
   "coinbasetxn": {

zebra-rpc/src/methods/tests/snapshot/snapshots/get_block_template@testnet_10.snap

@@ -6,14 +6,14 @@ expression: block_template
   "capabilities": [],
   "version": 4,
   "previousblockhash": "0000000000d723156d9b65ffcf4984da7a19675ed7e2f06d9e5d5188af087bf8",
-  "blockcommitmentshash": "0000000000000000000000000000000000000000000000000000000000000000",
-  "lightclientroothash": "0000000000000000000000000000000000000000000000000000000000000000",
-  "finalsaplingroothash": "0000000000000000000000000000000000000000000000000000000000000000",
+  "blockcommitmentshash": "cb1f1c6a5ad5ff9c4a170e3b747a24f3aec79817adba9a9451f19914481bb422",
+  "lightclientroothash": "cb1f1c6a5ad5ff9c4a170e3b747a24f3aec79817adba9a9451f19914481bb422",
+  "finalsaplingroothash": "cb1f1c6a5ad5ff9c4a170e3b747a24f3aec79817adba9a9451f19914481bb422",
   "defaultroots": {
     "merkleroot": "623400cc122baa015d3a4209f5903ebe215170c7e6e74831dce8372c5fd5b3cc",
-    "chainhistoryroot": "0000000000000000000000000000000000000000000000000000000000000000",
+    "chainhistoryroot": "03bc75f00c307a05aed2023819e18c2672cbe15fbd3200944997def141967387",
     "authdataroot": "a44375f0c0dd5ba612bd7b0efd77683cde8edf5055aff9fbfda443cc8d46bd3e",
-    "blockcommitmentshash": "0000000000000000000000000000000000000000000000000000000000000000"
+    "blockcommitmentshash": "cb1f1c6a5ad5ff9c4a170e3b747a24f3aec79817adba9a9451f19914481bb422"
   },
   "transactions": [],
   "coinbasetxn": {

zebra-rpc/src/methods/tests/utils.rs

@@ -0,0 +1,39 @@
+//! Utility functions for RPC method tests.
+
+use std::sync::Arc;
+use zebra_chain::{
+    block::Block,
+    history_tree::{HistoryTree, NonEmptyHistoryTree},
+    parameters::Network,
+    sapling::tree::Root,
+    serialization::ZcashDeserialize,
+};
+
+use zebra_test::vectors;
+
+/// Create a history tree with one single block for a network by using Zebra test vectors.
+pub fn fake_history_tree(network: Network) -> Arc<HistoryTree> {
+    let (block, sapling_root) = match network {
+        Network::Mainnet => (
+            &vectors::BLOCK_MAINNET_1046400_BYTES[..],
+            *vectors::SAPLING_FINAL_ROOT_MAINNET_1046400_BYTES,
+        ),
+        Network::Testnet => (
+            &vectors::BLOCK_TESTNET_1116000_BYTES[..],
+            *vectors::SAPLING_FINAL_ROOT_TESTNET_1116000_BYTES,
+        ),
+    };
+
+    let block = Arc::<Block>::zcash_deserialize(block).expect("block should deserialize");
+    let first_sapling_root = Root::try_from(sapling_root).unwrap();
+
+    let history_tree = NonEmptyHistoryTree::from_block(
+        Network::Mainnet,
+        block,
+        &first_sapling_root,
+        &Default::default(),
+    )
+    .unwrap();
+
+    Arc::new(HistoryTree::from(history_tree))
+}

zebra-rpc/src/methods/tests/vectors.rs

@@ -795,8 +795,11 @@ async fn rpc_getblocktemplate() {
 
     use chrono::{TimeZone, Utc};
 
-    use crate::methods::get_block_template_rpcs::constants::{
-        GET_BLOCK_TEMPLATE_MUTABLE_FIELD, GET_BLOCK_TEMPLATE_NONCE_RANGE_FIELD,
+    use crate::methods::{
+        get_block_template_rpcs::constants::{
+            GET_BLOCK_TEMPLATE_MUTABLE_FIELD, GET_BLOCK_TEMPLATE_NONCE_RANGE_FIELD,
+        },
+        tests::utils::fake_history_tree,
     };
     use zebra_chain::{
         amount::{Amount, NonNegative},
@@ -856,13 +859,15 @@ async fn rpc_getblocktemplate() {
             .clone()
             .expect_request_that(|req| matches!(req, ReadRequest::ChainInfo))
             .await
-            .respond(ReadResponse::ChainInfo(Some(GetBlockTemplateChainInfo {
+            .respond(ReadResponse::ChainInfo(GetBlockTemplateChainInfo {
                 expected_difficulty: CompactDifficulty::from(ExpandedDifficulty::from(U256::one())),
-                tip: (fake_tip_height, fake_tip_hash),
+                tip_height: fake_tip_height,
+                tip_hash: fake_tip_hash,
                 cur_time: fake_cur_time,
                 min_time: fake_min_time,
                 max_time: fake_max_time,
-            })));
+                history_tree: fake_history_tree(Mainnet),
+            }));
     });
 
     let get_block_template = tokio::spawn(get_block_template_rpc.get_block_template(None));

zebra-state/src/request.rs

@@ -61,8 +61,21 @@ impl From<block::Hash> for HashOrHeight {
 }
 
 impl From<block::Height> for HashOrHeight {
-    fn from(hash: block::Height) -> Self {
-        Self::Height(hash)
+    fn from(height: block::Height) -> Self {
+        Self::Height(height)
+    }
+}
+
+impl From<(block::Height, block::Hash)> for HashOrHeight {
+    fn from((_height, hash): (block::Height, block::Hash)) -> Self {
+        // Hash is more specific than height for the non-finalized state
+        hash.into()
+    }
+}
+
+impl From<(block::Hash, block::Height)> for HashOrHeight {
+    fn from((hash, _height): (block::Hash, block::Height)) -> Self {
+        hash.into()
     }
 }
 

zebra-state/src/response.rs

@@ -132,16 +132,20 @@ pub enum ReadResponse {
     #[cfg(feature = "getblocktemplate-rpcs")]
     /// Response to [`ReadRequest::ChainInfo`](crate::ReadRequest::ChainInfo) with the state
     /// information needed by the `getblocktemplate` RPC method.
-    ChainInfo(Option<GetBlockTemplateChainInfo>),
+    ChainInfo(GetBlockTemplateChainInfo),
 }
 
 #[cfg(feature = "getblocktemplate-rpcs")]
 /// A structure with the information needed from the state to build a `getblocktemplate` RPC response.
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub struct GetBlockTemplateChainInfo {
-    /// The current state tip height and hash.
-    /// The block template for the candidate block is the next block after this block.
-    pub tip: (block::Height, block::Hash),
+    /// The current state tip height.
+    /// The block template OAfor the candidate block is the next block after this block.
+    pub tip_height: block::Height,
+
+    /// The current state tip height.
+    /// The block template for the candidate block has this hash as the previous block hash.
+    pub tip_hash: block::Hash,
 
     /// The expected difficulty of the candidate block.
     pub expected_difficulty: CompactDifficulty,
@@ -154,6 +158,9 @@ pub struct GetBlockTemplateChainInfo {
 
     /// The maximum time the miner can use in this block.
     pub max_time: chrono::DateTime<chrono::Utc>,
+
+    /// The history tree of the current best chain.
+    pub history_tree: Arc<zebra_chain::history_tree::HistoryTree>,
 }
 
 /// Conversion from read-only [`ReadResponse`]s to read-write [`Response`]s.

zebra-state/src/service.rs

@@ -1607,21 +1607,16 @@ impl Service<ReadRequest> for ReadStateService {
                         // In that case, the `getblocktemplate` RPC will return an error because Zebra
                         // is not synced to the tip. That check happens before the RPC makes this request.
                         let get_block_template_info =
-                            read::tip(latest_non_finalized_state.best_chain(), &state.db).map(
-                                |tip| {
-                                    read::difficulty::difficulty_and_time_info(
-                                        &latest_non_finalized_state,
-                                        &state.db,
-                                        tip,
-                                        state.network,
-                                    )
-                                },
+                            read::difficulty::get_block_template_chain_info(
+                                &latest_non_finalized_state,
+                                &state.db,
+                                state.network,
                             );
 
                         // The work is done in the future.
                         timer.finish(module_path!(), line!(), "ReadRequest::ChainInfo");
 
-                        Ok(ReadResponse::ChainInfo(get_block_template_info))
+                        get_block_template_info.map(ReadResponse::ChainInfo)
                     })
                 })
                 .map(|join_result| join_result.expect("panic in ReadRequest::ChainInfo"))

zebra-state/src/service/read.rs

@@ -35,10 +35,17 @@ pub use block::{
 };
 
 #[cfg(feature = "getblocktemplate-rpcs")]
-pub use {block::hash, difficulty::difficulty_and_time_info};
+pub use {block::hash, difficulty::get_block_template_chain_info};
 
 pub use find::{
     best_tip, block_locator, chain_contains_hash, depth, find_chain_hashes, find_chain_headers,
     hash_by_height, height_by_hash, tip, tip_height,
 };
 pub use tree::{orchard_tree, sapling_tree};
+
+/// If a finalized state query is interrupted by a new finalized block,
+/// retry this many times.
+///
+/// Once we're at the tip, we expect up to 2 blocks to arrive at the same time.
+/// If any more arrive, the client should wait until we're synchronised with our peers.
+pub const FINALIZED_STATE_QUERY_RETRIES: usize = 3;

zebra-state/src/service/read/address.rs

@@ -3,10 +3,3 @@
 pub mod balance;
 pub mod tx_id;
 pub mod utxo;
-
-/// If the transparent address index queries are interrupted by a new finalized block,
-/// retry this many times.
-///
-/// Once we're at the tip, we expect up to 2 blocks to arrive at the same time.
-/// If any more arrive, the client should wait until we're synchronised with our peers.
-const FINALIZED_ADDRESS_INDEX_RETRIES: usize = 3;

zebra-state/src/service/read/address/balance.rs

@@ -20,12 +20,12 @@ use zebra_chain::{
 };
 
 use crate::{
-    service::{finalized_state::ZebraDb, non_finalized_state::Chain},
+    service::{
+        finalized_state::ZebraDb, non_finalized_state::Chain, read::FINALIZED_STATE_QUERY_RETRIES,
+    },
     BoxError,
 };
 
-use super::FINALIZED_ADDRESS_INDEX_RETRIES;
-
 /// Returns the total transparent balance for the supplied [`transparent::Address`]es.
 ///
 /// If the addresses do not exist in the non-finalized `chain` or finalized `db`, returns zero.
@@ -37,7 +37,9 @@ pub fn transparent_balance(
     let mut balance_result = finalized_transparent_balance(db, &addresses);
 
     // Retry the finalized balance query if it was interrupted by a finalizing block
-    for _ in 0..FINALIZED_ADDRESS_INDEX_RETRIES {
+    //
+    // TODO: refactor this into a generic retry(finalized_closure, process_and_check_closure) fn
+    for _ in 0..FINALIZED_STATE_QUERY_RETRIES {
         if balance_result.is_ok() {
             break;
         }