docs + more test cases + misc fixes

docs/docs/comment_dsl.mdx

@@ -104,6 +104,61 @@ foo = uint ; @newtype @custom_json
 
 Avoids generating and/or deriving json-related traits under the assumption that the user will supply their own implementation to be used in the generated library.
 
+## @custom_serialize / @custom_deserialize
+
+```cddl
+custom_bytes = bytes ; @custom_serialize custom_serialize_bytes @custom_deserialize custom_deserialize_bytes
+
+struct_with_custom_serialization = [
+  custom_bytes,
+  field: bytes, ; @custom_serialize custom_serialize_bytes @custom_deserialize custom_deserialize_bytes
+  overridden: custom_bytes, ; @custom_serialize write_hex_string @custom_deserialize read_hex_string
+  tagged1: #6.9(custom_bytes),
+  tagged2: #6.9(uint), ; @custom_serialize write_tagged_uint_str @custom_deserialize read_tagged_uint_str 
+]
+```
+
+This allows the overriding of serialization and/or deserialization for when a specific format must be maintained. This works even with primitives where _CDDL_CODEGEN_EXTERN_TYPE_ would require making a wrapper type to use.
+
+With `--preserve-encodings=true` the encoding variables must be passed in in the order they are used in cddl-codegen with regular serialization. They are passed in as `Option<cbor_event::Sz>` for integers/tags, `LenEncoding` for lengths and `StringEncoding` for text/bytes. These are the same types as are stored in the `*Encoding` structs generated. The same must be returned for deserialization. When there are no encoding variables the deserialized value should be directly returned, and if not a tuple with the value and its encoding variables should be returned.
+
+There are two ways to use this comment DSL:
+
+* Type level: e.g. `custom_bytes`. This will replace the (de)serialization everywhere you use this type.
+* Field level: e.g. `struct_with_custom_serialization.field`. This will entirely replace the (de)serialization logic for the entire field, including other encoding operations like tags, `.cbor`, etc.
+
+Example function signatures for `--preserve-encodings=false` for `custom_serialize_bytes` / `custom_deserialize_bytes` above:
+
+```rust
+pub fn custom_serialize_bytes<'se, W: std::io::Write>(
+    serializer: &'se mut cbor_event::se::Serializer<W>,
+    bytes: &[u8],
+) -> cbor_event::Result<&'se mut cbor_event::se::Serializer<W>>
+
+pub fn custom_deserialize_bytes<R: std::io::BufRead + std::io::Seek>(
+    raw: &mut cbor_event::de::Deserializer<R>,
+) -> Result<Vec<u8>, DeserializeError>
+```
+
+Example function signatures for `--preserve-encodings=true` for `write_tagged_uint_str` / `read_tagged_uint_str` above:
+
+```rust
+pub fn write_tagged_uint_str<'se, W: std::io::Write>(
+    serializer: &'se mut cbor_event::se::Serializer<W>,
+    uint: &u64,
+    tag_encoding: Option<cbor_event::Sz>,
+    text_encoding: Option<cbor_event::Sz>,
+) -> cbor_event::Result<&'se mut cbor_event::se::Serializer<W>>
+
+pub fn read_tagged_uint_str<R: std::io::BufRead + std::io::Seek>(
+    raw: &mut cbor_event::de::Deserializer<R>,
+) -> Result<(u64, Option<cbor_event::Sz>, Option<cbor_event::Sz>), DeserializeError>
+```
+
+Note that as this is at the field-level it must handle the tag as well as the `uint`.
+
+For more examples see `tests/custom_serialization` (used in the `core` and `core_no_wasm` tests) and `tests/custom_serialization_preserve` (used in the `preserve-encodings` test).
+
 ## _CDDL_CODEGEN_EXTERN_TYPE_
 
 While not as a comment, this allows you to compose in hand-written structs into a cddl spec.

src/generation.rs

@@ -2088,12 +2088,38 @@ impl GenerationScope {
         let deserializer_name = config.deserializer_name();
         // field-level @custom_deserialize overrides everything
         if let Some(custom_deserialize) = &config.custom_deserialize {
-            assert_eq!(config.final_exprs.len(), 0);
+            let deser_err_map = if config.final_exprs.is_empty() {
+                let enc_fields =
+                    encoding_fields_impl(types, config.var_name, serializing_rust_type, cli);
+                let (closure_args, tuple_fields) = if enc_fields.is_empty() {
+                    (config.var_name.to_owned(), "".to_owned())
+                } else {
+                    let enc_fields_names = enc_fields
+                        .iter()
+                        .map(|enc| enc.field_name.clone())
+                        .collect::<Vec<String>>()
+                        .join(", ");
+                    (
+                        format!("({}, {})", config.var_name, enc_fields_names),
+                        enc_fields_names,
+                    )
+                };
+                Cow::Owned(format!(
+                    ".map(|{}| ({}, {}, {}))",
+                    closure_args,
+                    config.var_name,
+                    config.final_exprs.join(", "),
+                    tuple_fields
+                ))
+            } else {
+                Cow::Borrowed("")
+            };
             deser_code.content.line(&format!(
-                "{}{}({}){}",
+                "{}{}({}){}{}",
                 before_after.before_str(true),
                 custom_deserialize,
                 deserializer_name,
+                deser_err_map,
                 before_after.after_str(true),
             ));
         } else {
@@ -4399,25 +4425,40 @@ struct EncodingField {
     /// this MUST be equivalent to the Default trait of the encoding field.
     /// This can be more concise though e.g. None for Option<T>::default()
     default_expr: &'static str,
+    enc_conversion_before: &'static str,
+    enc_conversion_after: &'static str,
     is_copy: bool,
     /// inner encodings - used for map/vec types
     #[allow(unused)]
     inner: Vec<EncodingField>,
 }
 
+impl EncodingField {
+    pub fn enc_conversion(&self, expr: &str) -> String {
+        format!(
+            "{}{}{}",
+            self.enc_conversion_before, expr, self.enc_conversion_after
+        )
+    }
+}
+
 fn key_encoding_field(name: &str, key: &FixedValue) -> EncodingField {
     match key {
         FixedValue::Text(_) => EncodingField {
             field_name: format!("{name}_key_encoding"),
             type_name: "StringEncoding".to_owned(),
             default_expr: "StringEncoding::default()",
+            enc_conversion_before: "StringEncoding::from(",
+            enc_conversion_after: ")",
             is_copy: false,
             inner: Vec::new(),
         },
         FixedValue::Uint(_) => EncodingField {
             field_name: format!("{name}_key_encoding"),
             type_name: "Option<cbor_event::Sz>".to_owned(),
             default_expr: "None",
+            enc_conversion_before: "Some(",
+            enc_conversion_after: ")",
             is_copy: true,
             inner: Vec::new(),
         },
@@ -4440,6 +4481,8 @@ fn encoding_fields(
             field_name: format!("{name}_default_present"),
             type_name: "bool".to_owned(),
             default_expr: "false",
+            enc_conversion_before: "",
+            enc_conversion_after: "",
             is_copy: true,
             inner: Vec::new(),
         });
@@ -4460,6 +4503,8 @@ fn encoding_fields_impl(
                 field_name: format!("{name}_encoding"),
                 type_name: "LenEncoding".to_owned(),
                 default_expr: "LenEncoding::default()",
+                enc_conversion_before: "",
+                enc_conversion_after: "",
                 is_copy: true,
                 inner: Vec::new(),
             };
@@ -4486,6 +4531,8 @@ fn encoding_fields_impl(
                         field_name: format!("{name}_elem_encodings"),
                         type_name: format!("Vec<{type_name_elem}>"),
                         default_expr: "Vec::new()",
+                        enc_conversion_before: "",
+                        enc_conversion_after: "",
                         is_copy: false,
                         inner: inner_encs,
                     },
@@ -4497,6 +4544,8 @@ fn encoding_fields_impl(
                 field_name: format!("{name}_encoding"),
                 type_name: "LenEncoding".to_owned(),
                 default_expr: "LenEncoding::default()",
+                enc_conversion_before: "",
+                enc_conversion_after: "",
                 is_copy: true,
                 inner: Vec::new(),
             }];
@@ -4525,6 +4574,8 @@ fn encoding_fields_impl(
                         type_name_value
                     ),
                     default_expr: "BTreeMap::new()",
+                    enc_conversion_before: "",
+                    enc_conversion_after: "",
                     is_copy: false,
                     inner: key_encs,
                 });
@@ -4551,6 +4602,8 @@ fn encoding_fields_impl(
                         type_name_value
                     ),
                     default_expr: "BTreeMap::new()",
+                    enc_conversion_before: "",
+                    enc_conversion_after: "",
                     is_copy: false,
                     inner: val_encs,
                 });
@@ -4562,6 +4615,8 @@ fn encoding_fields_impl(
                 field_name: format!("{name}_encoding"),
                 type_name: "StringEncoding".to_owned(),
                 default_expr: "StringEncoding::default()",
+                enc_conversion_before: "StringEncoding::from(",
+                enc_conversion_after: ")",
                 is_copy: false,
                 inner: Vec::new(),
             }],
@@ -4579,6 +4634,8 @@ fn encoding_fields_impl(
                 field_name: format!("{name}_encoding"),
                 type_name: "Option<cbor_event::Sz>".to_owned(),
                 default_expr: "None",
+                enc_conversion_before: "Some(",
+                enc_conversion_after: ")",
                 is_copy: true,
                 inner: Vec::new(),
             }],
@@ -5685,15 +5742,14 @@ fn codegen_struct(
                         }
                     }
                     if cli.preserve_encodings {
-                        let key_encoding_var = key_encoding_field(&field.name, &key).field_name;
-                        let enc_conversion = match &key {
-                            FixedValue::Uint(_) => "Some(key_enc)",
-                            FixedValue::Text(_) => "StringEncoding::from(key_enc)",
-                            _ => unimplemented!(),
-                        };
+                        let key_encoding = key_encoding_field(&field.name, &key);
                         deser_block_code
                             .content
-                            .line(&format!("{key_encoding_var} = {enc_conversion};"))
+                            .line(&format!(
+                                "{} = {};",
+                                key_encoding.field_name,
+                                key_encoding.enc_conversion("key_enc")
+                            ))
                             .line(&format!("orig_deser_order.push({field_index});"));
                     }
 
@@ -6398,6 +6454,8 @@ impl EnumVariantInRust {
                         field_name: "len_encoding".to_owned(),
                         type_name: "LenEncoding".to_owned(),
                         default_expr: "LenEncoding::default()",
+                        enc_conversion_before: "",
+                        enc_conversion_after: "",
                         is_copy: true,
                         inner: Vec::new(),
                     });
@@ -6425,6 +6483,8 @@ impl EnumVariantInRust {
                         field_name: "len_encoding".to_owned(),
                         type_name: "LenEncoding".to_owned(),
                         default_expr: "LenEncoding::default()",
+                        enc_conversion_before: "",
+                        enc_conversion_after: "",
                         is_copy: true,
                         inner: Vec::new(),
                     });

tests/core/input.cddl

@@ -200,8 +200,10 @@ enum_opt_embed_fields = [
 
 custom_bytes = bytes ; @custom_serialize custom_serialize_bytes @custom_deserialize custom_deserialize_bytes
 
-struct_with_custom_bytes = [
+struct_with_custom_serialization = [
   custom_bytes,
   field: bytes, ; @custom_serialize custom_serialize_bytes @custom_deserialize custom_deserialize_bytes
   overridden: custom_bytes, ; @custom_serialize write_hex_string @custom_deserialize read_hex_string
+  tagged1: #6.9(custom_bytes),
+  tagged2: #6.9(uint), ; @custom_serialize write_tagged_uint_str @custom_deserialize read_tagged_uint_str 
 ]

tests/core/tests.rs

@@ -470,19 +470,25 @@ mod tests {
 
     #[test]
     fn custom_serialization() {
-        let struct_with_custom_bytes = StructWithCustomBytes::new(
+        let struct_with_custom_bytes = StructWithCustomSerialization::new(
             vec![0xCA, 0xFE, 0xF0, 0x0D],
             vec![0x03, 0x01, 0x04, 0x01],
-            vec![0xBA, 0xAD, 0xD0, 0x0D]
+            vec![0xBA, 0xAD, 0xD0, 0x0D],
+            vec![0xDE, 0xAD, 0xBE, 0xEF],
+            1024,
         );
         use cbor_event::{Sz, StringLenSz};
         let bytes_special_enc = StringLenSz::Indefinite(vec![(1, Sz::Inline), (1, Sz::Inline), (1, Sz::Inline), (1, Sz::Inline)]);
         deser_test(&struct_with_custom_bytes);
         let expected_bytes = vec![
-            arr_def(3),
+            arr_def(5),
                 cbor_bytes_sz(vec![0xCA, 0xFE, 0xF0, 0x0D], bytes_special_enc.clone()),
                 cbor_bytes_sz(vec![0x03, 0x01, 0x04, 0x01], bytes_special_enc.clone()),
                 cbor_string("baadd00d"),
+                cbor_tag(9),
+                    cbor_bytes_sz(vec![0xDE, 0xAD, 0xBE, 0xEF], bytes_special_enc.clone()),
+                cbor_tag(9),
+                    cbor_string("1024")
         ].into_iter().flatten().clone().collect::<Vec<u8>>();
         assert_eq!(expected_bytes, struct_with_custom_bytes.to_cbor_bytes());
     }

tests/custom_serialization

@@ -43,3 +43,22 @@ pub fn read_hex_string<R: std::io::BufRead + std::io::Seek>(
     let text = raw.text()?;
     hex::decode(text).map_err(|e| DeserializeFailure::InvalidStructure(Box::new(e)).into())
 }
+
+// must include the tag since @custom_serialize at field-level overrides everything
+pub fn write_tagged_uint_str<'se, W: std::io::Write>(
+    serializer: &'se mut cbor_event::se::Serializer<W>,
+    uint: &u64,
+) -> cbor_event::Result<&'se mut cbor_event::se::Serializer<W>> {
+    serializer
+        .write_tag(9)?
+        .write_text(uint.to_string())
+}
+
+pub fn read_tagged_uint_str<R: std::io::BufRead + std::io::Seek>(
+    raw: &mut cbor_event::de::Deserializer<R>,
+) -> Result<u64, DeserializeError> {
+    use std::str::FromStr;
+    let tag = raw.tag()?;
+    let text = raw.text()?;
+    u64::from_str(&text).map_err(|e| DeserializeFailure::InvalidStructure(Box::new(e)).into())
+}

tests/custom_serialization_preserve

@@ -53,3 +53,34 @@ pub fn read_hex_string<R: std::io::BufRead + std::io::Seek>(
         .map(|bytes| (bytes, text_enc.into()))
         .map_err(|e| DeserializeFailure::InvalidStructure(Box::new(e)).into())
 }
+
+// must include the tag since @custom_serialize at field-level overrides everything
+pub fn write_tagged_uint_str<'se, W: std::io::Write>(
+    serializer: &'se mut cbor_event::se::Serializer<W>,
+    uint: &u64,
+    tag_encoding: Option<cbor_event::Sz>,
+    text_encoding: Option<cbor_event::Sz>,
+) -> cbor_event::Result<&'se mut cbor_event::se::Serializer<W>> {
+    let uint_string = uint.to_string();
+    let text_encoding = text_encoding
+        .map(|enc| crate::serialization::StringEncoding::Definite(enc))
+        .unwrap_or(crate::serialization::StringEncoding::Canonical);
+    let uint_string_encoding = text_encoding.to_str_len_sz(uint_string.len() as u64);
+    serializer
+        .write_tag_sz(9, fit_sz(9, tag_encoding))?
+        .write_text_sz(uint_string, uint_string_encoding)
+}
+
+pub fn read_tagged_uint_str<R: std::io::BufRead + std::io::Seek>(
+    raw: &mut cbor_event::de::Deserializer<R>,
+) -> Result<(u64, Option<cbor_event::Sz>, Option<cbor_event::Sz>), DeserializeError> {
+    use std::str::FromStr;
+    let (tag, tag_encoding) = raw.tag_sz()?;
+    let (text, text_encoding) = raw.text_sz()?;
+    match text_encoding {
+        cbor_event::StringLenSz::Indefinite(_) => Err(DeserializeFailure::CBOR(cbor_event::Error::CustomError(format!("We only support definite encodings in order to use the uint one"))).into()),
+        cbor_event::StringLenSz::Len(text_encoding_sz) => u64::from_str(&text)
+            .map(|uint| (uint, Some(tag_encoding), Some(text_encoding_sz)))
+            .map_err(|e| DeserializeFailure::InvalidStructure(Box::new(e)).into()),
+    }
+}

tests/preserve-encodings/input.cddl

@@ -149,8 +149,10 @@ enum_opt_embed_fields = [
 
 custom_bytes = bytes ; @custom_serialize custom_serialize_bytes @custom_deserialize custom_deserialize_bytes
 
-struct_with_custom_bytes = [
+struct_with_custom_serialization = [
   custom_bytes,
   field: bytes, ; @custom_serialize custom_serialize_bytes @custom_deserialize custom_deserialize_bytes
   overridden: custom_bytes, ; @custom_serialize write_hex_string @custom_deserialize read_hex_string
+  tagged1: #6.9(custom_bytes),
+  tagged2: #6.9(uint), ; @custom_serialize write_tagged_uint_str @custom_deserialize read_tagged_uint_str 
 ]

tests/preserve-encodings/tests.rs

@@ -1060,12 +1060,16 @@ mod tests {
             let bytes_special_enc = StringLenSz::Indefinite(vec![(1, *def_enc); 4]);
             for str_enc in &str_8_encodings {
                 let irregular_bytes = vec![
-                    arr_sz(3, *def_enc),
+                    arr_sz(5, *def_enc),
                         cbor_bytes_sz(vec![0xCA, 0xFE, 0xF0, 0x0D], bytes_special_enc.clone()),
                         cbor_bytes_sz(vec![0x03, 0x01, 0x04, 0x01], bytes_special_enc.clone()),
                         cbor_str_sz("baadd00d", str_enc.clone()),
+                        cbor_tag(9),
+                            cbor_bytes_sz(vec![0xDE, 0xAD, 0xBE, 0xEF], bytes_special_enc.clone()),
+                        cbor_tag(9),
+                            cbor_str_sz("10241024", StringLenSz::Len(*def_enc))
                 ].into_iter().flatten().clone().collect::<Vec<u8>>();
-                let from_bytes = StructWithCustomBytes::from_cbor_bytes(&irregular_bytes).unwrap();
+                let from_bytes = StructWithCustomSerialization::from_cbor_bytes(&irregular_bytes).unwrap();
                 assert_eq!(from_bytes.to_cbor_bytes(), irregular_bytes);
             }
         }