change(rpc): Adds a TrustedChainSync struct for keeping up with Zebra…

…'s non-finalized best chain from a separate process (#8596)

* Adds an init_read_only() fn in zebra-state

* moves elasticsearch initialization to `FinalizedState::new_with_debug()`

* Updates callers of `FinalizedState::{new, new_with_debug}` to pass a bool to try enabling elasticsearch

* Adds a non-finalized read state syncer to zebra-rpc

* moves, removes, updates, or addresses TODOs

* reduces disk IO while waiting for the a new chain tip & updates the chain tip sender when the finalized tip has changed.

* Returns boxed errors from RpcRequestClient methods instead of color_eyre type

* Avoids resetting the non-finalized state when there's an error getting a block unless it has the missing block error code.

* Adds stub for acceptance test(s) and removes outdated TODO

* adds TODOs for testing

* Tests that `ChainTipChange` is updated when the non-finalized best chain grows

* adds a last_chain_tip_hash and uses a FuturesOrdered for getblock requests

* Fixes a pre-flush sync issue by using a secondary db instead of a read-only db

* Moves disk IO to blocking tasks

* Updates acceptance test to how forks are handled

* Checks synced read state for all of the expected blocks

* checks that there isn't a tip change until the best chain changes

* checks for chain tip changes in test

* run test without feature

* fixes lint

* Fixes compilation/test issues

* Adds docs / comments, moves HexData out from behind the getblocktemplate-rpcs feature flag, moves test behind the mining feature flag.

* Fixes lints

* removes syncer and rpc-syncer features

* Fixes test on Windows, applies suggestions from code review

* Updates `POLL_DELAY` documentation

* Updates method docs

* Fixes a test bug

* use rpc-client feature in zebrad production code

* use rpc-client feature in zebra-node-services for building zebra-rpc crate

---------

Co-authored-by: Pili Guerra <mpguerra@users.noreply.github.com>
Co-authored-by: Alfredo Garcia <oxarbitrage@gmail.com>

zebra-node-services/src/rpc_client.rs

@@ -1,12 +1,12 @@
 //! A client for calling Zebra's JSON-RPC methods.
 //!
-//! Only used in tests and tools.
+//! Used in the rpc sync scanning functionality and in various tests and tools.
 
 use std::net::SocketAddr;
 
 use reqwest::Client;
 
-use color_eyre::{eyre::eyre, Result};
+use crate::BoxError;
 
 /// An HTTP client for making JSON-RPC requests.
 #[derive(Clone, Debug)]
@@ -99,21 +99,21 @@ impl RpcRequestClient {
         &self,
         method: impl AsRef<str>,
         params: impl AsRef<str>,
-    ) -> Result<T> {
+    ) -> std::result::Result<T, BoxError> {
         Self::json_result_from_response_text(&self.text_from_call(method, params).await?)
     }
 
     /// Accepts response text from an RPC call
     /// Returns `Ok` with a deserialized `result` value in the expected type, or an error report.
     fn json_result_from_response_text<T: serde::de::DeserializeOwned>(
         response_text: &str,
-    ) -> Result<T> {
+    ) -> std::result::Result<T, BoxError> {
         use jsonrpc_core::Output;
 
         let output: Output = serde_json::from_str(response_text)?;
         match output {
             Output::Success(success) => Ok(serde_json::from_value(success.result)?),
-            Output::Failure(failure) => Err(eyre!("RPC call failed with: {failure:?}")),
+            Output::Failure(failure) => Err(failure.error.into()),
         }
     }
 }

zebra-rpc/Cargo.toml

@@ -15,7 +15,6 @@ keywords = ["zebra", "zcash"]
 categories = ["asynchronous", "cryptography::cryptocurrencies", "encoding", "network-programming"]
 
 [features]
-default = []
 
 # Production features that activate extra dependencies, or extra features in dependencies
 
@@ -77,7 +76,7 @@ proptest = { version = "1.4.0", optional = true }
 zebra-chain = { path = "../zebra-chain", version = "1.0.0-beta.38", features = ["json-conversion"] }
 zebra-consensus = { path = "../zebra-consensus", version = "1.0.0-beta.38" }
 zebra-network = { path = "../zebra-network", version = "1.0.0-beta.38" }
-zebra-node-services = { path = "../zebra-node-services", version = "1.0.0-beta.38" }
+zebra-node-services = { path = "../zebra-node-services", version = "1.0.0-beta.38", features = ["rpc-client"] }
 zebra-script = { path = "../zebra-script", version = "1.0.0-beta.38" }
 zebra-state = { path = "../zebra-state", version = "1.0.0-beta.38" }
 

zebra-rpc/src/lib.rs

@@ -9,6 +9,7 @@ pub mod constants;
 pub mod methods;
 pub mod queue;
 pub mod server;
+pub mod sync;
 
 #[cfg(test)]
 mod tests;

zebra-rpc/src/methods.rs

@@ -38,6 +38,7 @@ use crate::{
 };
 
 mod errors;
+pub mod hex_data;
 
 use errors::{MapServerError, OkOrServerError};
 
@@ -171,6 +172,14 @@ pub trait Rpc {
     #[rpc(name = "getbestblockhash")]
     fn get_best_block_hash(&self) -> Result<GetBlockHash>;
 
+    /// Returns the height and hash of the current best blockchain tip block, as a [`GetBlockHeightAndHash`] JSON struct.
+    ///
+    /// zcashd reference: none
+    /// method: post
+    /// tags: blockchain
+    #[rpc(name = "getbestblockheightandhash")]
+    fn get_best_block_height_and_hash(&self) -> Result<GetBlockHeightAndHash>;
+
     /// Returns all transaction ids in the memory pool, as a JSON array.
     ///
     /// zcashd reference: [`getrawmempool`](https://zcash.github.io/rpc/getrawmempool.html)
@@ -867,15 +876,20 @@ where
         .boxed()
     }
 
-    // TODO: use a generic error constructor (#5548)
     fn get_best_block_hash(&self) -> Result<GetBlockHash> {
         self.latest_chain_tip
             .best_tip_hash()
             .map(GetBlockHash)
             .ok_or_server_error("No blocks in state")
     }
 
-    // TODO: use a generic error constructor (#5548)
+    fn get_best_block_height_and_hash(&self) -> Result<GetBlockHeightAndHash> {
+        self.latest_chain_tip
+            .best_tip_height_and_hash()
+            .map(|(height, hash)| GetBlockHeightAndHash { height, hash })
+            .ok_or_server_error("No blocks in state")
+    }
+
     fn get_raw_mempool(&self) -> BoxFuture<Result<Vec<String>>> {
         #[cfg(feature = "getblocktemplate-rpcs")]
         use zebra_chain::block::MAX_BLOCK_BYTES;
@@ -1547,6 +1561,24 @@ impl Default for GetBlock {
 #[serde(transparent)]
 pub struct GetBlockHash(#[serde(with = "hex")] pub block::Hash);
 
+/// Response to a `getbestblockheightandhash` RPC request.
+#[derive(Copy, Clone, Debug, Eq, PartialEq, serde::Deserialize, serde::Serialize)]
+pub struct GetBlockHeightAndHash {
+    /// The best chain tip block height
+    pub height: block::Height,
+    /// The best chain tip block hash
+    pub hash: block::Hash,
+}
+
+impl Default for GetBlockHeightAndHash {
+    fn default() -> Self {
+        Self {
+            height: block::Height::MIN,
+            hash: block::Hash([0; 32]),
+        }
+    }
+}
+
 impl Default for GetBlockHash {
     fn default() -> Self {
         GetBlockHash(block::Hash([0; 32]))

zebra-rpc/src/methods/get_block_template_rpcs.rs

@@ -46,15 +46,16 @@ use crate::methods::{
         types::{
             get_block_template::GetBlockTemplate,
             get_mining_info,
-            hex_data::HexData,
             long_poll::LongPollInput,
             peer_info::PeerInfo,
             submit_block,
             subsidy::{BlockSubsidy, FundingStream},
             unified_address, validate_address, z_validate_address,
         },
     },
-    height_from_signed_int, GetBlockHash, MISSING_BLOCK_ERROR_CODE,
+    height_from_signed_int,
+    hex_data::HexData,
+    GetBlockHash, MISSING_BLOCK_ERROR_CODE,
 };
 
 pub mod constants;

zebra-rpc/src/methods/get_block_template_rpcs/types.rs

@@ -3,7 +3,6 @@
 pub mod default_roots;
 pub mod get_block_template;
 pub mod get_mining_info;
-pub mod hex_data;
 pub mod long_poll;
 pub mod peer_info;
 pub mod submit_block;

zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template/parameters.rs

@@ -1,6 +1,6 @@
 //! Parameter types for the `getblocktemplate` RPC.
 
-use crate::methods::get_block_template_rpcs::types::{hex_data::HexData, long_poll::LongPollId};
+use crate::methods::{get_block_template_rpcs::types::long_poll::LongPollId, hex_data::HexData};
 
 /// Defines whether the RPC method should generate a block template or attempt to validate a block proposal.
 #[derive(Clone, Debug, serde::Deserialize, serde::Serialize, PartialEq, Eq)]

zebra-rpc/src/methods/get_block_template_rpcs/types/submit_block.rs

@@ -29,7 +29,7 @@ pub struct JsonParameters {
 /// Response to a `submitblock` RPC request.
 ///
 /// Zebra never returns "duplicate-invalid", because it does not store invalid blocks.
-#[derive(Debug, PartialEq, Eq, serde::Serialize)]
+#[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
 #[serde(rename_all = "kebab-case")]
 pub enum ErrorResponse {
     /// Block was already committed to the non-finalized or finalized state
@@ -45,7 +45,7 @@ pub enum ErrorResponse {
 /// Response to a `submitblock` RPC request.
 ///
 /// Zebra never returns "duplicate-invalid", because it does not store invalid blocks.
-#[derive(Debug, PartialEq, Eq, serde::Serialize)]
+#[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
 #[serde(untagged)]
 pub enum Response {
     /// Block was not successfully submitted, return error

zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs

@@ -39,13 +39,13 @@ use crate::methods::{
     get_block_template_rpcs::types::{
         get_block_template::{self, GetBlockTemplateRequestMode},
         get_mining_info,
-        hex_data::HexData,
         long_poll::{LongPollId, LONG_POLL_ID_LENGTH},
         peer_info::PeerInfo,
         submit_block,
         subsidy::BlockSubsidy,
         unified_address, validate_address, z_validate_address,
     },
+    hex_data::HexData,
     tests::{snapshot::EXCESSIVE_BLOCK_HEIGHT, utils::fake_history_tree},
     GetBlockHash, GetBlockTemplateRpc, GetBlockTemplateRpcImpl,
 };

zebra-rpc/src/methods/tests/vectors.rs

@@ -1259,8 +1259,9 @@ async fn rpc_getblocktemplate_mining_address(use_p2pkh: bool) {
                 GET_BLOCK_TEMPLATE_NONCE_RANGE_FIELD,
             },
             get_block_template::{self, GetBlockTemplateRequestMode},
-            types::{hex_data::HexData, long_poll::LONG_POLL_ID_LENGTH},
+            types::long_poll::LONG_POLL_ID_LENGTH,
         },
+        hex_data::HexData,
         tests::utils::fake_history_tree,
     };
 
@@ -1547,7 +1548,7 @@ async fn rpc_submitblock_errors() {
     use zebra_chain::chain_sync_status::MockSyncStatus;
     use zebra_network::address_book_peers::MockAddressBookPeers;
 
-    use crate::methods::get_block_template_rpcs::types::{hex_data::HexData, submit_block};
+    use crate::methods::{get_block_template_rpcs::types::submit_block, hex_data::HexData};
 
     let _init_guard = zebra_test::init();