feat(serde): show missed de fields in debug mode near#6436

chain/jsonrpc/client/src/lib.rs

@@ -12,6 +12,7 @@ use near_jsonrpc_primitives::types::changes::{
 };
 use near_jsonrpc_primitives::types::validator::RpcValidatorsOrderedRequest;
 use near_primitives::hash::CryptoHash;
+use near_primitives::serialize::DeserializedWithDebug;
 use near_primitives::types::{AccountId, BlockId, BlockReference, MaybeBlockId, ShardId};
 use near_primitives::views::validator_stake_view::ValidatorStakeView;
 use near_primitives::views::{
@@ -92,7 +93,11 @@ where
             response.body().map(|body| match body {
                 Ok(bytes) => String::from_utf8(bytes.to_vec())
                     .map_err(|err| format!("Error {:?} in {:?}", err, bytes))
-                    .and_then(|s| serde_json::from_str(&s).map_err(|err| err.to_string())),
+                    .and_then(|s| {
+                        serde_json::from_str::<DeserializedWithDebug<_>>(&s)
+                            .map(|v| v.0)
+                            .map_err(|err| err.to_string())
+                    }),
                 Err(_) => Err("Payload error: {:?}".to_string()),
             })
         })

core/chain-configs/src/genesis_config.rs

@@ -10,6 +10,7 @@ use std::{fmt, io};
 
 use anyhow::Context;
 use chrono::{DateTime, Utc};
+use near_primitives::serialize::DeserializedWithDebug;
 use num_rational::Rational;
 use serde::de::{self, DeserializeSeed, IgnoredAny, MapAccess, SeqAccess, Visitor};
 use serde::{Deserialize, Deserializer, Serialize};
@@ -291,7 +292,9 @@ impl GenesisConfig {
     ///
     /// It panics if the contents cannot be parsed from JSON to the GenesisConfig structure.
     pub fn from_json(value: &str) -> Self {
-        serde_json::from_str(value).expect("Failed to deserialize the genesis config.")
+        serde_json::from_str::<DeserializedWithDebug<_>>(value)
+            .expect("Failed to deserialize the genesis config.")
+            .0
     }
 
     /// Reads GenesisConfig from a JSON file.
@@ -341,7 +344,9 @@ impl GenesisRecords {
     ///
     /// It panics if the contents cannot be parsed from JSON to the GenesisConfig structure.
     pub fn from_json(value: &str) -> Self {
-        serde_json::from_str(value).expect("Failed to deserialize the genesis records.")
+        serde_json::from_str::<DeserializedWithDebug<_>>(value)
+            .expect("Failed to deserialize the genesis records.")
+            .0
     }
 
     /// Reads GenesisRecords from a JSON file.

core/crypto/Cargo.toml

@@ -31,6 +31,7 @@ subtle = "2.2"
 thiserror = "1"
 near-account-id = { path = "../account-id" }
 deepsize = { version = "0.2.0", optional = true }
+near-primitives-core = { path = "../primitives-core" }
 
 [dev-dependencies]
 hex-literal = "0.2"

core/crypto/src/key_file.rs

@@ -2,6 +2,7 @@ use std::fs::File;
 use std::io::{Read, Write};
 use std::path::Path;
 
+use near_primitives_core::serialize::DeserializedWithDebug;
 use serde::{Deserialize, Serialize};
 
 use crate::{PublicKey, SecretKey};
@@ -32,6 +33,8 @@ impl KeyFile {
         let mut file = File::open(path).expect("Could not open key file.");
         let mut content = String::new();
         file.read_to_string(&mut content).expect("Could not read from key file.");
-        serde_json::from_str(&content).expect("Failed to deserialize KeyFile")
+        serde_json::from_str::<DeserializedWithDebug<_>>(&content)
+            .expect("Failed to deserialize KeyFile")
+            .0
     }
 }

core/crypto/src/signature.rs

@@ -892,6 +892,8 @@ impl<'de> serde::Deserialize<'de> for Signature {
 
 #[cfg(test)]
 mod tests {
+    use near_primitives_core::serialize::DeserializedWithDebug;
+
     use super::*;
 
     #[test]
@@ -912,22 +914,29 @@ mod tests {
         let pk = sk.public_key();
         let expected = "\"ed25519:DcA2MzgpJbrUATQLLceocVckhhAqrkingax4oJ9kZ847\"";
         assert_eq!(serde_json::to_string(&pk).unwrap(), expected);
-        assert_eq!(pk, serde_json::from_str(expected).unwrap());
+        assert_eq!(pk, serde_json::from_str::<DeserializedWithDebug<_>>(expected).unwrap().0);
         assert_eq!(
             pk,
-            serde_json::from_str("\"DcA2MzgpJbrUATQLLceocVckhhAqrkingax4oJ9kZ847\"").unwrap()
+            serde_json::from_str::<DeserializedWithDebug<_>>(
+                "\"DcA2MzgpJbrUATQLLceocVckhhAqrkingax4oJ9kZ847\""
+            )
+            .unwrap()
+            .0
         );
         let pk2: PublicKey = pk.to_string().parse().unwrap();
         assert_eq!(pk, pk2);
 
         let expected = "\"ed25519:3KyUuch8pYP47krBq4DosFEVBMR5wDTMQ8AThzM8kAEcBQEpsPdYTZ2FPX5ZnSoLrerjwg66hwwJaW1wHzprd5k3\"";
         assert_eq!(serde_json::to_string(&sk).unwrap(), expected);
-        assert_eq!(sk, serde_json::from_str(expected).unwrap());
+        assert_eq!(sk, serde_json::from_str::<DeserializedWithDebug<_>>(expected).unwrap().0);
 
         let signature = sk.sign(b"123");
         let expected = "\"ed25519:3s1dvZdQtcAjBksMHFrysqvF63wnyMHPA4owNQmCJZ2EBakZEKdtMsLqrHdKWQjJbSRN6kRknN2WdwSBLWGCokXj\"";
         assert_eq!(serde_json::to_string(&signature).unwrap(), expected);
-        assert_eq!(signature, serde_json::from_str(expected).unwrap());
+        assert_eq!(
+            signature,
+            serde_json::from_str::<DeserializedWithDebug<_>>(expected).unwrap().0
+        );
         let signature_str: String = signature.to_string();
         let signature2: Signature = signature_str.parse().unwrap();
         assert_eq!(signature, signature2);
@@ -942,18 +951,21 @@ mod tests {
         let pk = sk.public_key();
         let expected = "\"secp256k1:BtJtBjukUQbcipnS78adSwUKE38sdHnk7pTNZH7miGXfodzUunaAcvY43y37nm7AKbcTQycvdgUzFNWsd7dgPZZ\"";
         assert_eq!(serde_json::to_string(&pk).unwrap(), expected);
-        assert_eq!(pk, serde_json::from_str(expected).unwrap());
+        assert_eq!(pk, serde_json::from_str::<DeserializedWithDebug<_>>(expected).unwrap().0);
         let pk2: PublicKey = pk.to_string().parse().unwrap();
         assert_eq!(pk, pk2);
 
         let expected = "\"secp256k1:9ZNzLxNff6ohoFFGkbfMBAFpZgD7EPoWeiuTpPAeeMRV\"";
         assert_eq!(serde_json::to_string(&sk).unwrap(), expected);
-        assert_eq!(sk, serde_json::from_str(expected).unwrap());
+        assert_eq!(sk, serde_json::from_str::<DeserializedWithDebug<_>>(expected).unwrap().0);
 
         let signature = sk.sign(&data);
         let expected = "\"secp256k1:7iA75xRmHw17MbUkSpHxBHFVTuJW6jngzbuJPJutwb3EAwVw21wrjpMHU7fFTAqH7D3YEma8utCdvdtsqcAWqnC7r\"";
         assert_eq!(serde_json::to_string(&signature).unwrap(), expected);
-        assert_eq!(signature, serde_json::from_str(expected).unwrap());
+        assert_eq!(
+            signature,
+            serde_json::from_str::<DeserializedWithDebug<_>>(expected).unwrap().0
+        );
         let signature_str: String = signature.to_string();
         let signature2: Signature = signature_str.parse().unwrap();
         assert_eq!(signature, signature2);
@@ -981,8 +993,8 @@ mod tests {
     #[test]
     fn test_invalid_data() {
         let invalid = "\"secp256k1:2xVqteU8PWhadHTv99TGh3bSf\"";
-        assert!(serde_json::from_str::<PublicKey>(invalid).is_err());
-        assert!(serde_json::from_str::<SecretKey>(invalid).is_err());
-        assert!(serde_json::from_str::<Signature>(invalid).is_err());
+        assert!(serde_json::from_str::<DeserializedWithDebug<PublicKey>>(invalid).is_err());
+        assert!(serde_json::from_str::<DeserializedWithDebug<SecretKey>>(invalid).is_err());
+        assert!(serde_json::from_str::<DeserializedWithDebug<Signature>>(invalid).is_err());
     }
 }

core/primitives-core/Cargo.toml

@@ -21,6 +21,8 @@ num-rational = { version = "0.3.1", features = ["serde"]}
 serde = { version = "1", features = ["derive"] }
 sha2 = "0.9"
 deepsize = { version = "0.2.0", optional = true }
+tracing = "0.1.13"
+serde_ignored = "0.1"
 
 near-account-id = { path = "../account-id" }
 

core/primitives-core/src/hash.rs

@@ -142,6 +142,8 @@ pub fn hash(data: &[u8]) -> CryptoHash {
 
 #[cfg(test)]
 mod tests {
+    use crate::serialize::DeserializedWithDebug;
+
     use super::*;
 
     #[derive(Deserialize, Serialize)]
@@ -167,22 +169,22 @@ mod tests {
     #[test]
     fn test_deserialize_default() {
         let encoded = "{\"hash\":\"11111111111111111111111111111111\"}";
-        let decoded: Struct = serde_json::from_str(encoded).unwrap();
+        let decoded: Struct = serde_json::from_str::<DeserializedWithDebug<_>>(encoded).unwrap().0;
         assert_eq!(decoded.hash, CryptoHash::default());
     }
 
     #[test]
     fn test_deserialize_success() {
         let encoded = "{\"hash\":\"CjNSmWXTWhC3EhRVtqLhRmWMTkRbU96wUACqxMtV1uGf\"}";
-        let decoded: Struct = serde_json::from_str(encoded).unwrap();
+        let decoded: Struct = serde_json::from_str::<DeserializedWithDebug<_>>(encoded).unwrap().0;
         assert_eq!(decoded.hash, hash(&[0, 1, 2]));
     }
 
     #[test]
     fn test_deserialize_not_base58() {
         let encoded = "\"---\"";
-        match serde_json::from_str(encoded) {
-            Ok(CryptoHash(_)) => assert!(false, "should have failed"),
+        match serde_json::from_str::<DeserializedWithDebug<CryptoHash>>(encoded) {
+            Ok(DeserializedWithDebug(_)) => assert!(false, "should have failed"),
             Err(_) => (),
         }
     }
@@ -196,7 +198,7 @@ mod tests {
             format!("\"{}\"", "1".repeat(33)),
             format!("\"{}\"", "1".repeat(1000)),
         ] {
-            match serde_json::from_str::<CryptoHash>(encoded) {
+            match serde_json::from_str::<DeserializedWithDebug<CryptoHash>>(encoded) {
                 Err(e) => if e.to_string() == "could not convert slice to array" {},
                 res => assert!(false, "should have failed with incorrect length error: {:?}", res),
             };

core/primitives-core/src/serialize.rs

@@ -1,3 +1,7 @@
+use tracing::debug;
+
+use serde::{Deserialize, Deserializer};
+
 pub fn to_base<T: AsRef<[u8]>>(input: T) -> String {
     bs58::encode(input).into_string()
 }
@@ -212,3 +216,27 @@ pub mod option_u128_dec_format {
         }
     }
 }
+
+#[derive(Debug, Deserialize)]
+pub struct DeserializedWithDebug<T>(pub T);
+
+impl<T> DeserializedWithDebug<T> {
+    fn _deserialize<'de, D>(deserializer: D) -> Result<T, D::Error>
+    where
+        T: Deserialize<'de>,
+        D: Deserializer<'de>,
+    {
+        if cfg!(debug_assertions) {
+            let mut unused_fields = Vec::new();
+            let de_result: Result<T, D::Error> = serde_ignored::deserialize(deserializer, |path| {
+                unused_fields.push(path.to_string());
+            });
+
+            debug!("missed fields while deserializing: {:?}", unused_fields);
+
+            return de_result;
+        }
+
+        <Self as Deserialize>::deserialize(deserializer).map(|v| v.0)
+    }
+}

core/primitives/src/runtime/config_store.rs

@@ -1,3 +1,5 @@
+use near_primitives_core::serialize::DeserializedWithDebug;
+
 use crate::runtime::config::RuntimeConfig;
 use crate::types::ProtocolVersion;
 use std::collections::BTreeMap;
@@ -44,7 +46,12 @@ impl RuntimeConfigStore {
     pub fn new(genesis_runtime_config: Option<&RuntimeConfig>) -> Self {
         let mut store =
             BTreeMap::from_iter(CONFIGS.iter().cloned().map(|(protocol_version, config_bytes)| {
-                (protocol_version, Arc::new(serde_json::from_slice(config_bytes).unwrap()))
+                (
+                    protocol_version,
+                    Arc::new(
+                        serde_json::from_slice::<DeserializedWithDebug<_>>(config_bytes).unwrap().0,
+                    ),
+                )
             }));
 
         if let Some(runtime_config) = genesis_runtime_config {
@@ -92,14 +99,17 @@ mod tests {
     use crate::version::ProtocolFeature::LowerDataReceiptAndEcrecoverBaseCost;
     use crate::version::ProtocolFeature::LowerStorageCost;
     use near_primitives_core::hash::hash;
+    use near_primitives_core::serialize::DeserializedWithDebug;
 
     const GENESIS_PROTOCOL_VERSION: ProtocolVersion = 29;
     const RECEIPTS_DEPTH: u64 = 63;
 
     fn check_config(protocol_version: ProtocolVersion, config_bytes: &[u8]) {
         assert_eq!(
             RuntimeConfigStore::new(None).get_config(protocol_version).as_ref(),
-            &serde_json::from_slice::<RuntimeConfig>(config_bytes).unwrap()
+            &serde_json::from_slice::<DeserializedWithDebug<RuntimeConfig>>(config_bytes)
+                .unwrap()
+                .0
         );
     }
 
@@ -203,14 +213,18 @@ If you add new config version, add a missing hash to the end of `expected_hashes
         assert_eq!(config.account_creation_config.min_allowed_top_level_account_length, 0);
         assert_ne!(
             config.as_ref(),
-            &serde_json::from_slice::<RuntimeConfig>(CONFIGS[1].1).unwrap()
+            &serde_json::from_slice::<DeserializedWithDebug<RuntimeConfig>>(CONFIGS[1].1)
+                .unwrap()
+                .0
         );
 
         let config = store.get_config(LowerDataReceiptAndEcrecoverBaseCost.protocol_version());
         assert_eq!(config.account_creation_config.min_allowed_top_level_account_length, 32);
         assert_eq!(
             config.as_ref(),
-            &serde_json::from_slice::<RuntimeConfig>(CONFIGS[2].1).unwrap()
+            &serde_json::from_slice::<DeserializedWithDebug<RuntimeConfig>>(CONFIGS[2].1)
+                .unwrap()
+                .0
         );
     }
 

core/primitives/src/serialize.rs

@@ -30,7 +30,8 @@ mod tests {
     #[test]
     fn test_deserialize_some() {
         let encoded = "{\"data\":\"ChQe\"}";
-        let decoded: OptionBytesStruct = serde_json::from_str(encoded).unwrap();
+        let decoded: OptionBytesStruct =
+            serde_json::from_str::<DeserializedWithDebug<_>>(encoded).unwrap().0;
         assert_eq!(decoded.data, Some(vec![10, 20, 30]));
     }
 
@@ -44,7 +45,8 @@ mod tests {
     #[test]
     fn test_deserialize_none() {
         let encoded = "{\"data\":null}";
-        let decoded: OptionBytesStruct = serde_json::from_str(encoded).unwrap();
+        let decoded: OptionBytesStruct =
+            serde_json::from_str::<DeserializedWithDebug<_>>(encoded).unwrap().0;
         assert_eq!(decoded.data, None);
     }
 
@@ -58,7 +60,8 @@ mod tests {
     #[test]
     fn test_deserialize_store_key() {
         let encoded = "{\"store_key\":\"ChQe\"}";
-        let decoded: StoreKeyStruct = serde_json::from_str(encoded).unwrap();
+        let decoded: StoreKeyStruct =
+            serde_json::from_str::<DeserializedWithDebug<_>>(encoded).unwrap().0;
         assert_eq!(decoded.store_key, StoreKey::from(vec![10, 20, 30]));
     }
 }