New multifile export format (#141)

New multifile export format

Assuming input directory contains: `lib.cddl`, `foo.cddl`, `bar.cddl`:

Old:
```
lib.rs
foo.rs
bar.rs
cbor_encodings.rs (for entire lib)
serialiation.rs (for entire lib)
```

New:
```
lib.rs
cbor_encodings.rs (for stuff in lib.rs only)
serialization.rs (for stuff in lib.rs only)

foo/mod.rs (old foo.rs)
foo/cbor_encodings.rs (for stuff in foo/mod.rs only)
foo/serialization.rs (for stuff in foo/mod.rs only)

bar/mod.rs
bar/cbor_encodings/rs
bar/serialization.rs
```

This change should help with organization, especially for users who
intend to put in hand-written code wrappers around the generated code
but want to do so in separate files e.g. `foo/utils.rs` for `foo`
wrappers and `bar/utils.rs`. This should also make it easier to see
changes in generation or do smaller generations as there is no longer
some 10k line serialization.rs containing the entire babbage.cddl spec
for example.

An extra test was added to test multi-file support with and without
preserve-encodings=true, etc as that can change which imports are
included.

GenerationScope::generate() / export() were also cleaned up to cleanly
separate imports/etc in one spot as well as remove any sort of
mutability in export(), so export() only exports (no extra
raw()/push_import() calls - all of this is done at the end of generate()
now).

src/main.rs

@@ -124,7 +124,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     println!("\n-----------------------------------------\n- Generating code...\n------------------------------------");
     let mut gen_scope = GenerationScope::new();
     gen_scope.generate(&types);
-    gen_scope.export(&types)?;
+    gen_scope.export()?;
     types.print_info();
 
     gen_scope.print_structs_without_deserialize();

src/test.rs

@@ -1,5 +1,7 @@
 use std::io::Write;
 
+/// If you have multiple tests that use the same directory, please use different export_suffix
+/// for each one or else the tests will be flaky as they are run concurrently.
 fn run_test(
     dir: &str,
     options: &[&str],
@@ -101,7 +103,18 @@ fn run_test(
             .unwrap();
         let extern_rs = std::fs::read_to_string(external_wasm_file_path).unwrap();
         wasm_lib_rs.write_all("\n\n".as_bytes()).unwrap();
-        wasm_lib_rs.write_all(extern_rs.as_bytes()).unwrap();
+        // we must replace the lib name if it's not the default
+        if let Some(custom_lib_name) = options.iter().find_map(|arg: &&str| {
+            arg.split_once("--lib-name=")
+                .map(|(_, lib_name)| lib_name.replace('-', "_"))
+        }) {
+            let replaced_extern_rs = extern_rs.replace("cddl_lib", &custom_lib_name);
+            wasm_lib_rs
+                .write_all(replaced_extern_rs.as_bytes())
+                .unwrap();
+        } else {
+            wasm_lib_rs.write_all(extern_rs.as_bytes()).unwrap();
+        }
     }
     if wasm_test_dir.exists() {
         println!("   ------ testing (wasm) ------");
@@ -234,10 +247,30 @@ fn multifile() {
     use std::str::FromStr;
     let extern_rust_path = std::path::PathBuf::from_str("tests")
         .unwrap()
-        .join("external_rust_defs_multifile");
+        .join("external_rust_defs");
     let extern_wasm_path = std::path::PathBuf::from_str("tests")
         .unwrap()
-        .join("external_wasm_defs_multifile");
+        .join("external_wasm_defs");
+    // this tests without preserve-encodings as that can affect imports
+    run_test(
+        "multifile",
+        &[],
+        None,
+        Some(extern_rust_path),
+        Some(extern_wasm_path),
+        true,
+    );
+}
+
+#[test]
+fn multifile_json_preserve() {
+    use std::str::FromStr;
+    let extern_rust_path = std::path::PathBuf::from_str("tests")
+        .unwrap()
+        .join("external_rust_defs_compiles_with_json_preserve");
+    let extern_wasm_path = std::path::PathBuf::from_str("tests")
+        .unwrap()
+        .join("external_wasm_defs");
     // json-schema-export / preserve-encodings to ensure that imports/scoping works in both:
     // 1) cbor_encodings.rs
     // 2) json-gen schema export crate
@@ -249,7 +282,7 @@ fn multifile() {
             "--json-serde-derives=true",
             "--json-schema-export=true",
         ],
-        None,
+        Some("json_preserve"),
         Some(extern_rust_path),
         Some(extern_wasm_path),
         true,

static/serialization_preserve.rs

@@ -50,7 +50,7 @@ pub trait DeserializeEmbeddedGroup {
 }
 
 #[inline]
-fn sz_max(sz: cbor_event::Sz) -> u64 {
+pub(crate) fn sz_max(sz: cbor_event::Sz) -> u64 {
     match sz {
         cbor_event::Sz::Inline => 23u64,
         cbor_event::Sz::One => u8::MAX as u64,

static/serialization_preserve_force_canonical.rs

@@ -1,5 +1,5 @@
 #[inline]
-fn fit_sz(len: u64, sz: Option<cbor_event::Sz>, force_canonical: bool) -> cbor_event::Sz {
+pub(crate) fn fit_sz(len: u64, sz: Option<cbor_event::Sz>, force_canonical: bool) -> cbor_event::Sz {
     match sz {
         Some(sz) => if !force_canonical && len <= sz_max(sz) {
             sz

static/serialization_preserve_non_force_canonical.rs

@@ -1,5 +1,5 @@
 #[inline]
-fn fit_sz(len: u64, sz: Option<cbor_event::Sz>) -> cbor_event::Sz {
+pub(crate) fn fit_sz(len: u64, sz: Option<cbor_event::Sz>) -> cbor_event::Sz {
     match sz {
         Some(sz) => if len <= sz_max(sz) {
             sz

tests/canonical/tests.rs

@@ -1,7 +1,8 @@
 #[cfg(test)]
 mod tests {
     use super::*;
-    use cbor_event::{Sz, StringLenSz};
+    use cbor_event::{de::Deserializer, Sz, StringLenSz};
+    use serialization::Deserialize;
 
     fn deser_test_orig<T: Deserialize + Serialize>(orig: &T) {
         print_cbor_types("orig (original enc)", &orig.to_cbor_bytes());

tests/core/tests.rs

@@ -1,6 +1,8 @@
 #[cfg(test)]
 mod tests {
     use super::*;
+    use cbor_event::de::Deserializer;
+    use serialization::Deserialize;
 
     fn deser_test<T: Deserialize + ToCBORBytes>(orig: &T) {
         let orig_bytes = orig.to_cbor_bytes();

tests/deser_test

@@ -14,13 +14,13 @@ fn map_def(len: u8) -> Vec<u8> {
 }
 
 fn arr_sz(len: u64, sz: cbor_event::Sz) -> Vec<u8> {
-    let mut buf = Serializer::new_vec();
+    let mut buf = cbor_event::se::Serializer::new_vec();
     buf.write_array_sz(cbor_event::LenSz::Len(len, sz)).unwrap();
     buf.finalize()
 }
 
 fn map_sz(len: u64, sz: cbor_event::Sz) -> Vec<u8> {
-    let mut buf = Serializer::new_vec();
+    let mut buf = cbor_event::se::Serializer::new_vec();
     buf.write_map_sz(cbor_event::LenSz::Len(len, sz)).unwrap();
     buf.finalize()
 }
@@ -40,7 +40,7 @@ fn cbor_tag(t: u8) -> Vec<u8> {
 }
 
 fn cbor_int(x: i128, sz: cbor_event::Sz) -> Vec<u8> {
-    let mut buf = Serializer::new_vec();
+    let mut buf = cbor_event::se::Serializer::new_vec();
     if x >= 0 {
         buf.write_unsigned_integer_sz(x as u64, sz).unwrap();
     } else {
@@ -50,26 +50,26 @@ fn cbor_int(x: i128, sz: cbor_event::Sz) -> Vec<u8> {
 }
 
 fn cbor_tag_sz(tag: u64, sz: cbor_event::Sz) -> Vec<u8> {
-    let mut buf = Serializer::new_vec();
+    let mut buf = cbor_event::se::Serializer::new_vec();
     buf.write_tag_sz(tag, sz).unwrap();
     buf.finalize()
 }
 
 fn cbor_str_sz(s: &str, sz: cbor_event::StringLenSz) -> Vec<u8> {
-    let mut buf = Serializer::new_vec();
+    let mut buf = cbor_event::se::Serializer::new_vec();
     buf.write_text_sz(s, sz).unwrap();
     buf.finalize()
 }
 
 fn cbor_bytes_sz(bytes: Vec<u8>, sz: cbor_event::StringLenSz) -> Vec<u8> {
-    let mut buf = Serializer::new_vec();
+    let mut buf = cbor_event::se::Serializer::new_vec();
     buf.write_bytes_sz(bytes, sz).unwrap();
     buf.finalize()
 }
 
 fn print_cbor_types(obj_name: &str, vec: &Vec<u8>) {
     use cbor_event::Type;
-    let mut raw = Deserializer::from(std::io::Cursor::new(vec));
+    let mut raw = cbor_event::de::Deserializer::from(std::io::Cursor::new(vec));
     let mut lens = Vec::new();
     let consume_elem = |lens: &mut Vec<cbor_event::LenSz>| {
         if let Some(len) = lens.last_mut() {

tests/external_rust_defs

@@ -16,10 +16,10 @@ impl ExternalFoo {
 }
 
 impl cbor_event::se::Serialize for ExternalFoo {
-    fn serialize<'se, W: Write>(
+    fn serialize<'se, W: std::io::Write>(
         &self,
-        serializer: &'se mut Serializer<W>,
-    ) -> cbor_event::Result<&'se mut Serializer<W>> {
+        serializer: &'se mut cbor_event::se::Serializer<W>,
+    ) -> cbor_event::Result<&'se mut cbor_event::se::Serializer<W>> {
         serializer.write_array(cbor_event::Len::Len(3))?;
         serializer.write_unsigned_integer(self.index_0)?;
         serializer.write_text(&self.index_1)?;
@@ -28,23 +28,23 @@ impl cbor_event::se::Serialize for ExternalFoo {
     }
 }
 
-impl Deserialize for ExternalFoo {
-    fn deserialize<R: BufRead + std::io::Seek>(raw: &mut Deserializer<R>) -> Result<Self, DeserializeError> {
+impl serialization::Deserialize for ExternalFoo {
+    fn deserialize<R: std::io::BufRead + std::io::Seek>(raw: &mut cbor_event::de::Deserializer<R>) -> Result<Self, error::DeserializeError> {
         let len = raw.array()?;
         let mut read_len = CBORReadLen::new(len);
         read_len.read_elems(3)?;
-        (|| -> Result<_, DeserializeError> {
+        (|| -> Result<_, error::DeserializeError> {
             let index_0 = Ok(raw.unsigned_integer()? as u64)
-                .map_err(|e: DeserializeError| e.annotate("index_0"))?;
+                .map_err(|e: error::DeserializeError| e.annotate("index_0"))?;
             let index_1 =
-                Ok(raw.text()? as String).map_err(|e: DeserializeError| e.annotate("index_1"))?;
+                Ok(raw.text()? as String).map_err(|e: error::DeserializeError| e.annotate("index_1"))?;
             let index_2 =
-                Ok(raw.bytes()? as Vec<u8>).map_err(|e: DeserializeError| e.annotate("index_2"))?;
+                Ok(raw.bytes()? as Vec<u8>).map_err(|e: error::DeserializeError| e.annotate("index_2"))?;
             match len {
                 cbor_event::Len::Len(_) => (),
                 cbor_event::Len::Indefinite => match raw.special()? {
                     cbor_event::Special::Break => (),
-                    _ => return Err(DeserializeFailure::EndingBreakMissing.into()),
+                    _ => return Err(error::DeserializeFailure::EndingBreakMissing.into()),
                 },
             }
             Ok(ExternalFoo {

tests/external_rust_defs_multifile → tests/external_rust_defs_compiles_with_json_preserve

@@ -1,5 +1,5 @@
-// doesn't actually preserve encodings - just different so that it compiles
-// also contains JSON schemas
+// same as external_rust_defs, but with JSON traits + serialization code compiles with --preserve-encodings=true
+// as this changes the API for CBORReadLen. This code does NOT support preserving encodings - just compiles with it.
 
 #[derive(Clone, Debug, serde::Deserialize, serde::Serialize, schemars::JsonSchema)]
 pub struct ExternalFoo {
@@ -19,10 +19,10 @@ impl ExternalFoo {
 }
 
 impl cbor_event::se::Serialize for ExternalFoo {
-    fn serialize<'se, W: Write>(
+    fn serialize<'se, W: std::io::Write>(
         &self,
-        serializer: &'se mut Serializer<W>,
-    ) -> cbor_event::Result<&'se mut Serializer<W>> {
+        serializer: &'se mut cbor_event::se::Serializer<W>,
+    ) -> cbor_event::Result<&'se mut cbor_event::se::Serializer<W>> {
         serializer.write_array(cbor_event::Len::Len(3))?;
         serializer.write_unsigned_integer(self.index_0)?;
         serializer.write_text(&self.index_1)?;
@@ -31,23 +31,23 @@ impl cbor_event::se::Serialize for ExternalFoo {
     }
 }
 
-impl Deserialize for ExternalFoo {
-    fn deserialize<R: BufRead + std::io::Seek>(raw: &mut Deserializer<R>) -> Result<Self, DeserializeError> {
+impl serialization::Deserialize for ExternalFoo {
+    fn deserialize<R: std::io::BufRead + std::io::Seek>(raw: &mut cbor_event::de::Deserializer<R>) -> Result<Self, error::DeserializeError> {
         let len = raw.array()?;
         let mut read_len = CBORReadLen::new(cbor_event::LenSz::Indefinite);
         read_len.read_elems(3)?;
-        (|| -> Result<_, DeserializeError> {
+        (|| -> Result<_, error::DeserializeError> {
             let index_0 = Ok(raw.unsigned_integer()? as u64)
-                .map_err(|e: DeserializeError| e.annotate("index_0"))?;
+                .map_err(|e: error::DeserializeError| e.annotate("index_0"))?;
             let index_1 =
-                Ok(raw.text()? as String).map_err(|e: DeserializeError| e.annotate("index_1"))?;
+                Ok(raw.text()? as String).map_err(|e: error::DeserializeError| e.annotate("index_1"))?;
             let index_2 =
-                Ok(raw.bytes()? as Vec<u8>).map_err(|e: DeserializeError| e.annotate("index_2"))?;
+                Ok(raw.bytes()? as Vec<u8>).map_err(|e: error::DeserializeError| e.annotate("index_2"))?;
             match len {
                 cbor_event::Len::Len(_) => (),
                 cbor_event::Len::Indefinite => match raw.special()? {
                     cbor_event::Special::Break => (),
-                    _ => return Err(DeserializeFailure::EndingBreakMissing.into()),
+                    _ => return Err(error::DeserializeFailure::EndingBreakMissing.into()),
                 },
             }
             Ok(ExternalFoo {

tests/preserve-encodings/tests.rs

@@ -1,7 +1,8 @@
 #[cfg(test)]
 mod tests {
     use super::*;
-    use cbor_event::{Sz, StringLenSz};
+    use cbor_event::{de::Deserializer, Sz, StringLenSz};
+    use serialization::Deserialize;
 
     fn deser_test<T: Deserialize + ToCBORBytes>(orig: &T) {
         print_cbor_types("orig", &orig.to_cbor_bytes());

tests/rust-wasm-split/tests.rs

@@ -1,6 +1,8 @@
 #[cfg(test)]
 mod tests {
     use super::*;
+    use cbor_event::de::Deserializer;
+    use serialization::Deserialize;
 
     fn deser_test<T: Deserialize + ToCBORBytes>(orig: &T) {
         print_cbor_types("orig", &orig.to_cbor_bytes());