Merge branch 'alex/efficient-byte-ser-in-rpc-for-vault-2' into 'master'

perf(crypto): CRP-2210 efficient byte vector serialization in RPC with the crypto vault

This is a duplicate of the reverted !15222. See the respective MR description and !15614 for more details. 

See merge request dfinity-lab/public/ic!15764

rs/crypto/internal/crypto_service_provider/src/vault/remote_csp_vault/mod.rs

@@ -29,6 +29,7 @@ use ic_types::crypto::canister_threshold_sig::{
 };
 use ic_types::crypto::{AlgorithmId, CurrentNodePublicKeys};
 use ic_types::{NodeId, NodeIndex, NumberOfNodes, Randomness};
+use serde_bytes::ByteBuf;
 use std::collections::{BTreeMap, BTreeSet};
 use std::path::Path;
 use tokio::net::UnixListener;
@@ -65,7 +66,7 @@ pub trait TarpcCspVault {
     // Corresponds to `BasicSignatureCspVault.sign()`.
     async fn sign(
         algorithm_id: AlgorithmId,
-        message: Vec<u8>,
+        message: ByteBuf,
         key_id: KeyId,
     ) -> Result<CspSignature, CspBasicSignatureError>;
 
@@ -75,7 +76,7 @@ pub trait TarpcCspVault {
     // Corresponds to `MultiSignatureCspVault.multi_sign()`.
     async fn multi_sign(
         algorithm_id: AlgorithmId,
-        message: Vec<u8>,
+        message: ByteBuf,
         key_id: KeyId,
     ) -> Result<CspSignature, CspMultiSignatureError>;
 
@@ -86,7 +87,7 @@ pub trait TarpcCspVault {
     // Corresponds to `ThresholdSignatureCspVault.threshold_sign()`.
     async fn threshold_sign(
         algorithm_id: AlgorithmId,
-        message: Vec<u8>,
+        message: ByteBuf,
         key_id: KeyId,
     ) -> Result<CspSignature, CspThresholdSignError>;
 
@@ -155,13 +156,13 @@ pub trait TarpcCspVault {
     ) -> Result<TlsPublicKeyCert, CspTlsKeygenError>;
 
     // Corresponds to `TlsHandshakeCspVault.tls_sign()`.
-    async fn tls_sign(message: Vec<u8>, key_id: KeyId) -> Result<CspSignature, CspTlsSignError>;
+    async fn tls_sign(message: ByteBuf, key_id: KeyId) -> Result<CspSignature, CspTlsSignError>;
 
     // Corresponds to `IDkgProtocolCspVault.idkg_create_dealing`
     #[allow(clippy::too_many_arguments)]
     async fn idkg_create_dealing(
         algorithm_id: AlgorithmId,
-        context_data: Vec<u8>,
+        context_data: ByteBuf,
         dealer_index: NodeIndex,
         reconstruction_threshold: NumberOfNodes,
         receiver_keys: Vec<MEGaPublicKey>,
@@ -175,13 +176,13 @@ pub trait TarpcCspVault {
         dealer_index: NodeIndex,
         receiver_index: NodeIndex,
         receiver_key_id: KeyId,
-        context_data: Vec<u8>,
+        context_data: ByteBuf,
     ) -> Result<(), IDkgVerifyDealingPrivateError>;
 
     // Corresponds to `IDkgProtocolCspVault.idkg_load_transcript`
     async fn idkg_load_transcript(
         dealings: BTreeMap<NodeIndex, BatchSignedIDkgDealing>,
-        context_data: Vec<u8>,
+        context_data: ByteBuf,
         receiver_index: NodeIndex,
         key_id: KeyId,
         transcript: IDkgTranscriptInternalBytes,
@@ -192,7 +193,7 @@ pub trait TarpcCspVault {
     async fn idkg_load_transcript_with_openings(
         dealings: BTreeMap<NodeIndex, BatchSignedIDkgDealing>,
         openings: BTreeMap<NodeIndex, BTreeMap<NodeIndex, CommitmentOpening>>,
-        context_data: Vec<u8>,
+        context_data: ByteBuf,
         receiver_index: NodeIndex,
         key_id: KeyId,
         transcript: IDkgTranscriptInternalBytes,
@@ -211,7 +212,7 @@ pub trait TarpcCspVault {
     async fn idkg_open_dealing(
         dealing: IDkgDealingInternal,
         dealer_index: NodeIndex,
-        context_data: Vec<u8>,
+        context_data: ByteBuf,
         opener_index: NodeIndex,
         opener_key_id: KeyId,
     ) -> Result<CommitmentOpening, IDkgOpenTranscriptError>;
@@ -220,7 +221,7 @@ pub trait TarpcCspVault {
     #[allow(clippy::too_many_arguments)]
     async fn ecdsa_sign_share(
         derivation_path: ExtendedDerivationPath,
-        hashed_message: Vec<u8>,
+        hashed_message: ByteBuf,
         nonce: Randomness,
         key_raw: IDkgTranscriptInternalBytes,
         kappa_unmasked_raw: IDkgTranscriptInternalBytes,

rs/crypto/internal/crypto_service_provider/src/vault/remote_csp_vault/tarpc_csp_vault_client.rs

@@ -46,6 +46,7 @@ use ic_types::crypto::canister_threshold_sig::{
 use ic_types::crypto::{AlgorithmId, CurrentNodePublicKeys};
 use ic_types::{NodeId, NumberOfNodes, Randomness};
 use serde::{Deserialize, Serialize};
+use serde_bytes::ByteBuf;
 use std::collections::{BTreeMap, BTreeSet};
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
@@ -253,7 +254,7 @@ impl BasicSignatureCspVault for RemoteCspVault {
         self.tokio_block_on(self.tarpc_csp_client.sign(
             context_with_timeout(self.rpc_timeout),
             algorithm_id,
-            message.to_vec(),
+            ByteBuf::from(message),
             key_id,
         ))
         .unwrap_or_else(|rpc_error: tarpc::client::RpcError| {
@@ -286,7 +287,7 @@ impl MultiSignatureCspVault for RemoteCspVault {
         self.tokio_block_on(self.tarpc_csp_client.multi_sign(
             context_with_timeout(self.rpc_timeout),
             algorithm_id,
-            message.to_vec(),
+            ByteBuf::from(message),
             key_id,
         ))
         .unwrap_or_else(|rpc_error: tarpc::client::RpcError| {
@@ -321,7 +322,7 @@ impl ThresholdSignatureCspVault for RemoteCspVault {
         self.tokio_block_on(self.tarpc_csp_client.threshold_sign(
             context_with_timeout(self.rpc_timeout),
             algorithm_id,
-            message.to_vec(),
+            ByteBuf::from(message),
             key_id,
         ))
         .unwrap_or_else(|rpc_error: tarpc::client::RpcError| {
@@ -551,7 +552,7 @@ impl TlsHandshakeCspVault for RemoteCspVault {
         tokio::task::block_in_place(|| {
             self.tokio_block_on(self.tarpc_csp_client.tls_sign(
                 context_with_timeout(self.rpc_timeout),
-                message.to_vec(),
+                ByteBuf::from(message),
                 *key_id,
             ))
             .unwrap_or_else(|rpc_error: tarpc::client::RpcError| {
@@ -576,7 +577,7 @@ impl IDkgProtocolCspVault for RemoteCspVault {
         self.tokio_block_on(self.tarpc_csp_client.idkg_create_dealing(
             context_with_timeout(self.rpc_timeout),
             algorithm_id,
-            context_data.to_vec(),
+            ByteBuf::from(context_data),
             dealer_index,
             reconstruction_threshold,
             receiver_keys.to_vec(),
@@ -605,7 +606,7 @@ impl IDkgProtocolCspVault for RemoteCspVault {
             dealer_index,
             receiver_index,
             receiver_key_id,
-            context_data.to_vec(),
+            ByteBuf::from(context_data),
         ))
         .unwrap_or_else(|rpc_error: tarpc::client::RpcError| {
             Err(IDkgVerifyDealingPrivateError::TransientInternalError {
@@ -625,7 +626,7 @@ impl IDkgProtocolCspVault for RemoteCspVault {
         self.tokio_block_on(self.tarpc_csp_client.idkg_load_transcript(
             context_with_timeout(self.rpc_timeout),
             dealings.clone(),
-            context_data.to_vec(),
+            ByteBuf::from(context_data),
             receiver_index,
             *key_id,
             transcript.clone(),
@@ -650,7 +651,7 @@ impl IDkgProtocolCspVault for RemoteCspVault {
             context_with_timeout(self.rpc_timeout),
             dealings.clone(),
             openings.clone(),
-            context_data.to_vec(),
+            ByteBuf::from(context_data),
             receiver_index,
             *key_id,
             transcript.clone(),
@@ -703,7 +704,7 @@ impl IDkgProtocolCspVault for RemoteCspVault {
             context_with_timeout(self.rpc_timeout),
             dealing,
             dealer_index,
-            context_data.to_vec(),
+            ByteBuf::from(context_data),
             opener_index,
             *opener_key_id,
         ))
@@ -732,7 +733,7 @@ impl ThresholdEcdsaSignerCspVault for RemoteCspVault {
         self.tokio_block_on(self.tarpc_csp_client.ecdsa_sign_share(
             context_with_timeout(self.rpc_timeout),
             derivation_path.clone(),
-            hashed_message.to_vec(),
+            ByteBuf::from(hashed_message),
             *nonce,
             key_raw,
             kappa_unmasked_raw,

rs/crypto/internal/crypto_service_provider/src/vault/remote_csp_vault/tarpc_csp_vault_server.rs

@@ -42,6 +42,7 @@ use ic_types::crypto::canister_threshold_sig::{
 };
 use ic_types::crypto::{AlgorithmId, CurrentNodePublicKeys};
 use ic_types::{NodeId, NumberOfNodes, Randomness};
+use serde_bytes::ByteBuf;
 use std::collections::{BTreeMap, BTreeSet};
 use std::path::Path;
 use std::sync::Arc;
@@ -116,7 +117,7 @@ impl<C: CspVault + 'static> TarpcCspVault for TarpcCspVaultServerWorker<C> {
         self,
         _: context::Context,
         algorithm_id: AlgorithmId,
-        msg: Vec<u8>,
+        msg: ByteBuf,
         key_id: KeyId,
     ) -> Result<CspSignature, CspBasicSignatureError> {
         let vault = self.local_csp_vault;
@@ -138,7 +139,7 @@ impl<C: CspVault + 'static> TarpcCspVault for TarpcCspVaultServerWorker<C> {
         self,
         _: context::Context,
         algorithm_id: AlgorithmId,
-        message: Vec<u8>,
+        message: ByteBuf,
         key_id: KeyId,
     ) -> Result<CspSignature, CspMultiSignatureError> {
         let vault = self.local_csp_vault;
@@ -160,7 +161,7 @@ impl<C: CspVault + 'static> TarpcCspVault for TarpcCspVaultServerWorker<C> {
         self,
         _: context::Context,
         algorithm_id: AlgorithmId,
-        message: Vec<u8>,
+        message: ByteBuf,
         key_id: KeyId,
     ) -> Result<CspSignature, CspThresholdSignError> {
         let vault = self.local_csp_vault;
@@ -318,7 +319,7 @@ impl<C: CspVault + 'static> TarpcCspVault for TarpcCspVaultServerWorker<C> {
     async fn tls_sign(
         self,
         _: context::Context,
-        message: Vec<u8>,
+        message: ByteBuf,
         key_id: KeyId,
     ) -> Result<CspSignature, CspTlsSignError> {
         let vault = self.local_csp_vault;
@@ -331,7 +332,7 @@ impl<C: CspVault + 'static> TarpcCspVault for TarpcCspVaultServerWorker<C> {
         self,
         _: context::Context,
         algorithm_id: AlgorithmId,
-        context_data: Vec<u8>,
+        context_data: ByteBuf,
         dealer_index: NodeIndex,
         reconstruction_threshold: NumberOfNodes,
         receiver_keys: Vec<MEGaPublicKey>,
@@ -359,7 +360,7 @@ impl<C: CspVault + 'static> TarpcCspVault for TarpcCspVaultServerWorker<C> {
         dealer_index: NodeIndex,
         receiver_index: NodeIndex,
         receiver_key_id: KeyId,
-        context_data: Vec<u8>,
+        context_data: ByteBuf,
     ) -> Result<(), IDkgVerifyDealingPrivateError> {
         let vault = self.local_csp_vault;
         let job = move || {
@@ -379,7 +380,7 @@ impl<C: CspVault + 'static> TarpcCspVault for TarpcCspVaultServerWorker<C> {
         self,
         _: context::Context,
         dealings: BTreeMap<NodeIndex, BatchSignedIDkgDealing>,
-        context_data: Vec<u8>,
+        context_data: ByteBuf,
         receiver_index: NodeIndex,
         key_id: KeyId,
         transcript: IDkgTranscriptInternalBytes,
@@ -402,7 +403,7 @@ impl<C: CspVault + 'static> TarpcCspVault for TarpcCspVaultServerWorker<C> {
         _: context::Context,
         dealings: BTreeMap<NodeIndex, BatchSignedIDkgDealing>,
         openings: BTreeMap<NodeIndex, BTreeMap<NodeIndex, CommitmentOpening>>,
-        context_data: Vec<u8>,
+        context_data: ByteBuf,
         receiver_index: NodeIndex,
         key_id: KeyId,
         transcript: IDkgTranscriptInternalBytes,
@@ -446,7 +447,7 @@ impl<C: CspVault + 'static> TarpcCspVault for TarpcCspVaultServerWorker<C> {
         _: context::Context,
         dealing: IDkgDealingInternal,
         dealer_index: NodeIndex,
-        context_data: Vec<u8>,
+        context_data: ByteBuf,
         opener_index: NodeIndex,
         opener_key_id: KeyId,
     ) -> Result<CommitmentOpening, IDkgOpenTranscriptError> {
@@ -468,7 +469,7 @@ impl<C: CspVault + 'static> TarpcCspVault for TarpcCspVaultServerWorker<C> {
         self,
         _: context::Context,
         derivation_path: ExtendedDerivationPath,
-        hashed_message: Vec<u8>,
+        hashed_message: ByteBuf,
         nonce: Randomness,
         key_raw: IDkgTranscriptInternalBytes,
         kappa_unmasked_raw: IDkgTranscriptInternalBytes,