Initial pull method

pallets/manta-pay/Cargo.toml

@@ -107,17 +107,17 @@ sp-std = { git = 'https://github.com/paritytech/substrate.git', branch = "polkad
 jsonrpsee = { version = "0.15.0", features = ["server", "macros"], optional = true }
 
 # manta dependencies
-manta-accounting = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", default-features = false }
-manta-crypto = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", default-features = false }
-manta-pay = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", default-features = false, features = ["groth16", "parameters", "scale"] }
+manta-accounting = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.13", default-features = false }
+manta-crypto = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.13", default-features = false }
+manta-pay = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.13", default-features = false, features = ["groth16", "parameters", "scale"] }
 manta-primitives = { path = "../../primitives/manta", default-features = false }
 manta-support = { package = "pallet-manta-support", path = "../manta-support", default-features = false }
-manta-util = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", default-features = false }
+manta-util = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.13", default-features = false }
 
 [dev-dependencies]
 lazy_static = "1.4.0"
-manta-crypto = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", features = ["getrandom"] }
-manta-pay = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", features = ["groth16", "parameters", "scale", "download", "test"] }
+manta-crypto = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.13", features = ["getrandom"] }
+manta-pay = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.13", features = ["groth16", "parameters", "scale", "download", "test"] }
 pallet-asset-manager = { path = "../asset-manager" }
 pallet-assets = { git = 'https://github.com/paritytech/substrate.git', branch = "polkadot-v0.9.28" }
 pallet-balances = { git = 'https://github.com/paritytech/substrate.git', branch = "polkadot-v0.9.28" }

pallets/manta-pay/src/lib.rs

@@ -83,13 +83,13 @@ use manta_pay::{
 use manta_primitives::assets::{self, AssetConfig, FungibleLedger as _};
 use manta_support::manta_pay::{
     asset_value_decode, asset_value_encode, fp_decode, fp_encode, id_from_field, AccountId, Asset,
-    AssetValue, FullIncomingNote, MTParametersError, NullifierCommitment, OutgoingNote,
-    ReceiverChunk, SenderChunk, StandardAssetId, TransferPost, Utxo, UtxoAccumulatorOutput,
-    UtxoItemHashError, UtxoMerkleTreePath, VerifyingContextError, Wrap, WrapPair,
+    AssetValue, Checkpoint, FullIncomingNote, InitialSyncResponse, MTParametersError,
+    NullifierCommitment, OutgoingNote, PullResponse, ReceiverChunk, SenderChunk, StandardAssetId,
+    TransferPost, Utxo, UtxoAccumulatorOutput, UtxoItemHashError, UtxoMerkleTreePath,
+    VerifyingContextError, Wrap, WrapPair,
 };
 use manta_util::codec::Encode;
 
-pub use manta_support::manta_pay::{Checkpoint, PullResponse, RawCheckpoint};
 pub use pallet::*;
 pub use weights::WeightInfo;
 
@@ -559,6 +559,25 @@ pub mod pallet {
             Shards::<T>::contains_key(shard_index, max_receiver_index)
         }
 
+        /// Returns the diff of ledger state since the given `checkpoint` and `max_receivers` to
+        /// perform the initial synchronization.
+        #[inline]
+        pub fn initial_pull(checkpoint: Checkpoint, max_receivers: u64) -> InitialSyncResponse {
+            let (should_continue, receivers) =
+                Self::pull_receivers(*checkpoint.receiver_index, max_receivers);
+            let utxo_data = receivers.into_iter().map(|receiver| receiver.0).collect();
+            let membership_proof_data = (0..=255)
+                .map(|i| ShardTrees::<T>::get(i).current_path)
+                .collect();
+            let nullifier_count = NullifierSetSize::<T>::get() as u128;
+            InitialSyncResponse {
+                should_continue,
+                utxo_data,
+                membership_proof_data,
+                nullifier_count,
+            }
+        }
+
         /// Pulls sender data from the ledger starting at the `sender_index`.
         #[inline]
         fn pull_senders(sender_index: usize, max_update_request: u64) -> (bool, SenderChunk) {

pallets/manta-pay/src/rpc.rs

@@ -16,15 +16,17 @@
 
 //! MantaPay RPC Interfaces
 
-use crate::{runtime::PullLedgerDiffApi, Checkpoint, PullResponse};
+use crate::runtime::PullLedgerDiffApi;
 use alloc::sync::Arc;
 use core::marker::PhantomData;
 use jsonrpsee::{
     core::{async_trait, RpcResult},
     proc_macros::rpc,
     types::error::{CallError, ErrorObject},
 };
-use manta_support::manta_pay::DensePullResponse;
+use manta_support::manta_pay::{
+    Checkpoint, DenseInitialSyncResponse, DensePullResponse, InitialSyncResponse, PullResponse,
+};
 use sp_api::ProvideRuntimeApi;
 use sp_blockchain::HeaderBackend;
 use sp_runtime::{generic::BlockId, traits::Block};
@@ -52,6 +54,21 @@ pub trait PullApi {
         max_receivers: u64,
         max_senders: u64,
     ) -> RpcResult<DensePullResponse>;
+
+    /// Returns the update required for the initial synchronization with the ledger.
+    #[method(name = "mantaPay_initial_pull", blocking)]
+    fn initial_pull(
+        &self,
+        checkpoint: Checkpoint,
+        max_receivers: u64,
+    ) -> RpcResult<InitialSyncResponse>;
+
+    #[method(name = "mantaPay_dense_initial_pull", blocking)]
+    fn dense_initial_pull(
+        &self,
+        checkpoint: Checkpoint,
+        max_receivers: u64,
+    ) -> RpcResult<DenseInitialSyncResponse>;
 }
 
 /// Pull RPC API Implementation
@@ -121,4 +138,43 @@ where
                 .into()
             })
     }
+
+    #[inline]
+    fn initial_pull(
+        &self,
+        checkpoint: Checkpoint,
+        max_receivers: u64,
+    ) -> RpcResult<InitialSyncResponse> {
+        let api = self.client.runtime_api();
+        let at = BlockId::hash(self.client.info().finalized_hash);
+        api.initial_pull(&at, checkpoint.into(), max_receivers)
+            .map_err(|err| {
+                CallError::Custom(ErrorObject::owned(
+                    PULL_LEDGER_DIFF_ERROR,
+                    "Unable to compute state diff for initial pull",
+                    Some(format!("{err:?}")),
+                ))
+                .into()
+            })
+    }
+
+    #[inline]
+    fn dense_initial_pull(
+        &self,
+        checkpoint: Checkpoint,
+        max_receivers: u64,
+    ) -> RpcResult<DenseInitialSyncResponse> {
+        let api = self.client.runtime_api();
+        let at = BlockId::hash(self.client.info().finalized_hash);
+        api.initial_pull(&at, checkpoint.into(), max_receivers)
+            .map(Into::into)
+            .map_err(|err| {
+                CallError::Custom(ErrorObject::owned(
+                    PULL_LEDGER_DIFF_ERROR,
+                    "Unable to compute state diff for initial pull",
+                    Some(format!("{err:?}")),
+                ))
+                .into()
+            })
+    }
 }

pallets/manta-pay/src/runtime.rs

@@ -16,10 +16,11 @@
 
 //! MantaPay Runtime APIs
 
-use crate::{PullResponse, RawCheckpoint};
+use manta_support::manta_pay::{InitialSyncResponse, PullResponse, RawCheckpoint};
 
 sp_api::decl_runtime_apis! {
     pub trait PullLedgerDiffApi {
         fn pull_ledger_diff(checkpoint: RawCheckpoint, max_receivers: u64, max_senders: u64) -> PullResponse;
+        fn initial_pull(checkpoint: RawCheckpoint, max_receivers: u64) -> InitialSyncResponse;
     }
 }

pallets/manta-sbt/Cargo.toml

@@ -101,17 +101,17 @@ sp-std = { git = 'https://github.com/paritytech/substrate.git', branch = "polkad
 jsonrpsee = { version = "0.15.0", features = ["server", "macros"], optional = true }
 
 # manta dependencies
-manta-accounting = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", default-features = false }
-manta-crypto = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", default-features = false }
-manta-pay = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", default-features = false, features = ["groth16", "parameters", "scale"] }
+manta-accounting = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.13", default-features = false }
+manta-crypto = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.13", default-features = false }
+manta-pay = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.13", default-features = false, features = ["groth16", "parameters", "scale"] }
 manta-primitives = { path = "../../primitives/manta", default-features = false }
 manta-support = { package = "pallet-manta-support", path = "../manta-support", default-features = false }
-manta-util = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", default-features = false }
+manta-util = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.13", default-features = false }
 
 [dev-dependencies]
 lazy_static = "1.4.0"
-manta-crypto = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", features = ["getrandom"] }
-manta-pay = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", features = ["groth16", "parameters", "scale", "download", "test"] }
+manta-crypto = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.13", features = ["getrandom"] }
+manta-pay = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.13", features = ["groth16", "parameters", "scale", "download", "test"] }
 pallet-asset-manager = { path = "../asset-manager" }
 pallet-assets = { git = "https://github.com/paritytech/substrate.git", branch = "polkadot-v0.9.28" }
 pallet-balances = { git = 'https://github.com/paritytech/substrate.git', branch = "polkadot-v0.9.28" }

pallets/manta-support/Cargo.toml

@@ -29,11 +29,11 @@ sp-runtime = { git = "https://github.com/paritytech/substrate.git", branch = "po
 sp-std = { git = 'https://github.com/paritytech/substrate.git', branch = "polkadot-v0.9.28", default-features = false }
 
 # manta dependencies
-manta-accounting = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", default-features = false }
-manta-crypto = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", default-features = false }
-manta-pay = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", default-features = false, features = ["groth16", "parameters", "scale"] }
+manta-accounting = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.13", default-features = false }
+manta-crypto = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.13", default-features = false }
+manta-pay = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.13", default-features = false, features = ["groth16", "parameters", "scale"] }
 manta-primitives = { path = "../../primitives/manta", default-features = false }
-manta-util = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.12", default-features = false }
+manta-util = { git = "https://github.com/manta-network/manta-rs.git", tag = "v0.5.13", default-features = false }
 
 [features]
 default = ["std"]

pallets/manta-support/src/manta_pay.rs

@@ -17,7 +17,6 @@
 //! Type Definitions for Manta Pay
 
 use alloc::{boxed::Box, vec::Vec};
-use manta_crypto::merkle_tree;
 use manta_pay::{
     config::{
         self,
@@ -26,6 +25,7 @@ use manta_pay::{
     crypto::poseidon::encryption::{self, BlockArray, CiphertextBlock},
     manta_crypto::{
         encryption::{hybrid, EmptyHeader},
+        merkle_tree,
         permutation::duplex,
         signature::schnorr,
     },
@@ -825,7 +825,12 @@ pub type LeafDigest = [u8; 32];
 pub type InnerDigest = [u8; 32];
 
 /// Merkle Tree Current Path
-#[derive(Clone, Debug, Decode, Default, Encode, Eq, PartialEq, TypeInfo)]
+#[cfg_attr(
+    feature = "serde",
+    derive(Deserialize, Serialize),
+    serde(crate = "manta_util::serde", deny_unknown_fields)
+)]
+#[derive(Clone, Debug, Decode, Default, Encode, Eq, Hash, PartialEq, TypeInfo)]
 pub struct CurrentPath {
     /// Sibling Digest
     pub sibling_digest: LeafDigest,
@@ -899,9 +904,77 @@ pub struct UtxoMerkleTreePath {
 /// Receiver Chunk Data Type
 pub type ReceiverChunk = Vec<(Utxo, FullIncomingNote)>;
 
+/// Utxo Chunk Data Type
+pub type UtxoChunk = Vec<Utxo>;
+
+/// Merkle Tree [`CurrentPath`] Chunk Data Type
+pub type CurrentPathChunk = Vec<CurrentPath>;
+
 /// Sender Chunk Data Type
 pub type SenderChunk = Vec<(NullifierCommitment, OutgoingNote)>;
 
+/// Initial Sync Response
+#[cfg_attr(
+    feature = "serde",
+    derive(Deserialize, Serialize),
+    serde(crate = "manta_util::serde", deny_unknown_fields)
+)]
+#[derive(Clone, Debug, Decode, Default, Encode, Eq, Hash, PartialEq, TypeInfo)]
+pub struct InitialSyncResponse {
+    /// Initial Sync Continuation Flag
+    ///
+    /// The `should_continue` flag is set to `true` if the client should request more data from the
+    /// ledger to finish the pull.
+    pub should_continue: bool,
+
+    /// Ledger Utxo Chunk
+    pub utxo_data: UtxoChunk,
+
+    /// Ledger [`CurrentPath`] Chunk
+    pub membership_proof_data: CurrentPathChunk,
+
+    /// Nullifier Count
+    pub nullifier_count: u128,
+}
+
+/// Ledger Source Dense Pull Response
+#[cfg_attr(
+    feature = "serde",
+    derive(Deserialize, Serialize),
+    serde(crate = "manta_util::serde", deny_unknown_fields)
+)]
+#[derive(Clone, Debug, Encode, Default, Eq, Hash, Decode, PartialEq, TypeInfo)]
+pub struct DenseInitialSyncResponse {
+    /// Pull Continuation Flag
+    ///
+    /// The `should_continue` flag is set to `true` if the client should request more data from the
+    /// ledger to finish the pull.
+    pub should_continue: bool,
+
+    /// Ledger Utxo Chunk
+    #[codec(skip)]
+    pub utxo_data: alloc::string::String,
+
+    /// Ledger [`CurrentPath`] Chunk
+    #[codec(skip)]
+    pub membership_proof_data: alloc::string::String,
+
+    /// Nullifier Count
+    pub nullifier_count: u128,
+}
+
+impl From<InitialSyncResponse> for DenseInitialSyncResponse {
+    #[inline]
+    fn from(resp: InitialSyncResponse) -> DenseInitialSyncResponse {
+        Self {
+            should_continue: resp.should_continue,
+            utxo_data: base64::encode(resp.utxo_data.encode()),
+            membership_proof_data: base64::encode(resp.membership_proof_data.encode()),
+            nullifier_count: resp.nullifier_count,
+        }
+    }
+}
+
 /// Ledger Source Pull Response
 #[cfg_attr(
     feature = "serde",

runtime/calamari/src/lib.rs

@@ -56,7 +56,7 @@ use manta_primitives::{
     constants::{time::*, RocksDbWeight, STAKING_PALLET_ID, TREASURY_PALLET_ID, WEIGHT_PER_SECOND},
     types::{AccountId, Balance, BlockNumber, Hash, Header, Index, Signature},
 };
-use manta_support::manta_pay::{PullResponse, RawCheckpoint};
+use manta_support::manta_pay::{InitialSyncResponse, PullResponse, RawCheckpoint};
 pub use pallet_parachain_staking::{InflationInfo, Range};
 use pallet_session::ShouldEndSession;
 use runtime_common::{
@@ -1070,6 +1070,9 @@ impl_runtime_apis! {
         ) -> PullResponse {
             MantaPay::pull_ledger_diff(checkpoint.into(), max_receiver, max_sender)
         }
+        fn initial_pull(checkpoint: RawCheckpoint, max_receiver: u64) -> InitialSyncResponse {
+            MantaPay::initial_pull(checkpoint.into(), max_receiver)
+        }
     }
 
     impl pallet_manta_sbt::runtime::SBTPullLedgerDiffApi<Block> for Runtime {