Initial pull method

pallets/manta-pay/Cargo.toml

@@ -107,16 +107,16 @@ sp-std = { git = 'https://github.com/paritytech/substrate.git', branch = "polkad
 jsonrpsee = { version = "0.15.0", features = ["server", "macros"], optional = true }
 
 # manta dependencies
-manta-accounting = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", default-features = false }
-manta-crypto = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", default-features = false }
-manta-pay = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", default-features = false, features = ["groth16", "parameters", "scale"] }
+manta-accounting = { git = "https://github.com/manta-network/manta-rs.git", branch = "signer_initial_sync_checkout", default-features = false }
+manta-crypto = { git = "https://github.com/manta-network/manta-rs.git", branch = "signer_initial_sync_checkout", default-features = false }
+manta-pay = { git = "https://github.com/manta-network/manta-rs.git", branch = "signer_initial_sync_checkout", default-features = false, features = ["groth16", "parameters", "scale"] }
 manta-primitives = { path = "../../primitives/manta", default-features = false }
-manta-util = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", default-features = false }
+manta-util = { git = "https://github.com/manta-network/manta-rs.git", branch = "signer_initial_sync_checkout", default-features = false }
 
 [dev-dependencies]
 lazy_static = "1.4.0"
-manta-crypto = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", features = ["getrandom"] }
-manta-pay = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", features = ["groth16", "parameters", "scale", "download", "test"] }
+manta-crypto = { git = "https://github.com/manta-network/manta-rs.git", branch = "signer_initial_sync_checkout", features = ["getrandom"] }
+manta-pay = { git = "https://github.com/manta-network/manta-rs.git", branch = "signer_initial_sync_checkout", features = ["groth16", "parameters", "scale", "download", "test"] }
 pallet-asset-manager = { path = "../asset-manager" }
 pallet-assets = { git = 'https://github.com/paritytech/substrate.git', branch = "polkadot-v0.9.28" }
 pallet-balances = { git = 'https://github.com/paritytech/substrate.git', branch = "polkadot-v0.9.28" }

pallets/manta-pay/src/lib.rs

@@ -90,7 +90,7 @@ use manta_util::{
 
 pub use crate::types::{Checkpoint, RawCheckpoint};
 pub use pallet::*;
-pub use types::PullResponse;
+pub use types::{InitialSyncResponse, PullResponse};
 pub use weights::WeightInfo;
 
 #[cfg(test)]
@@ -565,6 +565,25 @@ pub mod pallet {
             Shards::<T>::contains_key(shard_index, max_receiver_index)
         }
 
+        /// Returns the diff of ledger state since the given `checkpoint` and `max_receivers` to
+        /// perform the initial synchronization.
+        #[inline]
+        pub fn initial_pull(checkpoint: Checkpoint, max_receivers: u64) -> InitialSyncResponse {
+            let (should_continue, receivers) =
+                Self::pull_receivers(*checkpoint.receiver_index, max_receivers);
+            let utxo_data = receivers.into_iter().map(|receiver| receiver.0).collect();
+            let membership_proof_data = (0..=255)
+                .map(|i| ShardTrees::<T>::get(i).current_path)
+                .collect();
+            let nullifier_count = NullifierSetSize::<T>::get() as u128;
+            InitialSyncResponse {
+                should_continue,
+                utxo_data,
+                membership_proof_data,
+                nullifier_count,
+            }
+        }
+
         /// Pulls sender data from the ledger starting at the `sender_index`.
         #[inline]
         fn pull_senders(sender_index: usize, max_update_request: u64) -> (bool, SenderChunk) {

pallets/manta-pay/src/rpc.rs

@@ -16,7 +16,11 @@
 
 //! MantaPay RPC Interfaces
 
-use crate::{runtime::PullLedgerDiffApi, types::DensePullResponse, Checkpoint, PullResponse};
+use crate::{
+    runtime::PullLedgerDiffApi,
+    types::{DenseInitialSyncResponse, DensePullResponse},
+    Checkpoint, InitialSyncResponse, PullResponse,
+};
 use alloc::sync::Arc;
 use core::marker::PhantomData;
 use jsonrpsee::{
@@ -51,6 +55,21 @@ pub trait PullApi {
         max_receivers: u64,
         max_senders: u64,
     ) -> RpcResult<DensePullResponse>;
+
+    /// Returns the update required for the initial synchronization with the ledger.
+    #[method(name = "mantaPay_initial_pull", blocking)]
+    fn initial_pull(
+        &self,
+        checkpoint: Checkpoint,
+        max_receivers: u64,
+    ) -> RpcResult<InitialSyncResponse>;
+
+    #[method(name = "mantaPay_dense_initial_pull", blocking)]
+    fn dense_initial_pull(
+        &self,
+        checkpoint: Checkpoint,
+        max_receivers: u64,
+    ) -> RpcResult<DenseInitialSyncResponse>;
 }
 
 /// Pull RPC API Implementation
@@ -120,4 +139,43 @@ where
                 .into()
             })
     }
+
+    #[inline]
+    fn initial_pull(
+        &self,
+        checkpoint: Checkpoint,
+        max_receivers: u64,
+    ) -> RpcResult<InitialSyncResponse> {
+        let api = self.client.runtime_api();
+        let at = BlockId::hash(self.client.info().finalized_hash);
+        api.initial_pull(&at, checkpoint.into(), max_receivers)
+            .map_err(|err| {
+                CallError::Custom(ErrorObject::owned(
+                    PULL_LEDGER_DIFF_ERROR,
+                    "Unable to compute state diff for initial pull",
+                    Some(format!("{err:?}")),
+                ))
+                .into()
+            })
+    }
+
+    #[inline]
+    fn dense_initial_pull(
+        &self,
+        checkpoint: Checkpoint,
+        max_receivers: u64,
+    ) -> RpcResult<DenseInitialSyncResponse> {
+        let api = self.client.runtime_api();
+        let at = BlockId::hash(self.client.info().finalized_hash);
+        api.initial_pull(&at, checkpoint.into(), max_receivers)
+            .map(Into::into)
+            .map_err(|err| {
+                CallError::Custom(ErrorObject::owned(
+                    PULL_LEDGER_DIFF_ERROR,
+                    "Unable to compute state diff for initial pull",
+                    Some(format!("{err:?}")),
+                ))
+                .into()
+            })
+    }
 }

pallets/manta-pay/src/runtime.rs

@@ -16,10 +16,11 @@
 
 //! MantaPay Runtime APIs
 
-use crate::{PullResponse, RawCheckpoint};
+use crate::{InitialSyncResponse, PullResponse, RawCheckpoint};
 
 sp_api::decl_runtime_apis! {
     pub trait PullLedgerDiffApi {
         fn pull_ledger_diff(checkpoint: RawCheckpoint, max_receivers: u64, max_senders: u64) -> PullResponse;
+        fn initial_pull(checkpoint: RawCheckpoint, max_receivers: u64) -> InitialSyncResponse;
     }
 }

pallets/manta-pay/src/types.rs

@@ -17,7 +17,6 @@
 //! Type Definitions for Manta Pay
 
 use alloc::{boxed::Box, vec::Vec};
-use manta_crypto::merkle_tree;
 use manta_pay::{
     config::{
         self,
@@ -26,6 +25,7 @@ use manta_pay::{
     crypto::poseidon::encryption::{self, BlockArray, CiphertextBlock},
     manta_crypto::{
         encryption::{hybrid, EmptyHeader},
+        merkle_tree,
         permutation::duplex,
         signature::schnorr,
     },
@@ -803,7 +803,12 @@ pub type LeafDigest = [u8; 32];
 pub type InnerDigest = [u8; 32];
 
 /// Merkle Tree Current Path
-#[derive(Clone, Debug, Decode, Default, Encode, Eq, PartialEq, TypeInfo)]
+#[cfg_attr(
+    feature = "serde",
+    derive(Deserialize, Serialize),
+    serde(crate = "manta_util::serde", deny_unknown_fields)
+)]
+#[derive(Clone, Debug, Decode, Default, Encode, Eq, Hash, PartialEq, TypeInfo)]
 pub struct CurrentPath {
     /// Sibling Digest
     pub sibling_digest: LeafDigest,
@@ -877,9 +882,77 @@ pub struct UtxoMerkleTreePath {
 /// Receiver Chunk Data Type
 pub type ReceiverChunk = Vec<(Utxo, FullIncomingNote)>;
 
+/// Utxo Chunk Data Type
+pub type UtxoChunk = Vec<Utxo>;
+
+/// Merkle Tree [`CurrentPath`] Chunk Data Type
+pub type CurrentPathChunk = Vec<CurrentPath>;
+
 /// Sender Chunk Data Type
 pub type SenderChunk = Vec<(NullifierCommitment, OutgoingNote)>;
 
+/// Initial Sync Response
+#[cfg_attr(
+    feature = "serde",
+    derive(Deserialize, Serialize),
+    serde(crate = "manta_util::serde", deny_unknown_fields)
+)]
+#[derive(Clone, Debug, Decode, Default, Encode, Eq, Hash, PartialEq, TypeInfo)]
+pub struct InitialSyncResponse {
+    /// Initial Sync Continuation Flag
+    ///
+    /// The `should_continue` flag is set to `true` if the client should request more data from the
+    /// ledger to finish the pull.
+    pub should_continue: bool,
+
+    /// Ledger Utxo Chunk
+    pub utxo_data: UtxoChunk,
+
+    /// Ledger [`CurrentPath`] Chunk
+    pub membership_proof_data: CurrentPathChunk,
+
+    /// Nullifier Count
+    pub nullifier_count: u128,
+}
+
+/// Ledger Source Dense Pull Response
+#[cfg_attr(
+    feature = "serde",
+    derive(Deserialize, Serialize),
+    serde(crate = "manta_util::serde", deny_unknown_fields)
+)]
+#[derive(Clone, Debug, Encode, Default, Eq, Hash, Decode, PartialEq, TypeInfo)]
+pub struct DenseInitialSyncResponse {
+    /// Pull Continuation Flag
+    ///
+    /// The `should_continue` flag is set to `true` if the client should request more data from the
+    /// ledger to finish the pull.
+    pub should_continue: bool,
+
+    /// Ledger Utxo Chunk
+    #[codec(skip)]
+    pub utxo_data: alloc::string::String,
+
+    /// Ledger [`CurrentPath`] Chunk
+    #[codec(skip)]
+    pub membership_proof_data: alloc::string::String,
+
+    /// Nullifier Count
+    pub nullifier_count: u128,
+}
+
+impl From<InitialSyncResponse> for DenseInitialSyncResponse {
+    #[inline]
+    fn from(resp: InitialSyncResponse) -> DenseInitialSyncResponse {
+        Self {
+            should_continue: resp.should_continue,
+            utxo_data: base64::encode(resp.utxo_data.encode()),
+            membership_proof_data: base64::encode(resp.membership_proof_data.encode()),
+            nullifier_count: resp.nullifier_count,
+        }
+    }
+}
+
 /// Ledger Source Pull Response
 #[cfg_attr(
     feature = "serde",

runtime/calamari/src/lib.rs

@@ -1066,6 +1066,9 @@ impl_runtime_apis! {
         ) -> pallet_manta_pay::PullResponse {
             MantaPay::pull_ledger_diff(checkpoint.into(), max_receiver, max_sender)
         }
+        fn initial_pull(checkpoint: pallet_manta_pay::RawCheckpoint, max_receiver: u64) -> pallet_manta_pay::InitialSyncResponse {
+            MantaPay::initial_pull(checkpoint.into(), max_receiver)
+        }
     }
 
     impl nimbus_primitives::NimbusApi<Block> for Runtime {

runtime/dolphin/src/lib.rs

@@ -976,6 +976,9 @@ impl_runtime_apis! {
         ) -> pallet_manta_pay::PullResponse {
             MantaPay::pull_ledger_diff(checkpoint.into(), max_receiver, max_sender)
         }
+        fn initial_pull(checkpoint: pallet_manta_pay::RawCheckpoint, max_receiver: u64) -> pallet_manta_pay::InitialSyncResponse {
+            MantaPay::initial_pull(checkpoint.into(), max_receiver)
+        }
     }
 
     impl nimbus_primitives::NimbusApi<Block> for Runtime {