From 38a32926ed192f434758052c4090268effad30c8 Mon Sep 17 00:00:00 2001 From: raphael-goetz Date: Sat, 1 Nov 2025 10:44:55 +0100 Subject: [PATCH 1/4] feat: included de/serialization into module export --- build/rust/src/lib.rs | 74 ++----------------------------------------- 1 file changed, 3 insertions(+), 71 deletions(-) diff --git a/build/rust/src/lib.rs b/build/rust/src/lib.rs index 143b7bb..36d8bc2 100644 --- a/build/rust/src/lib.rs +++ b/build/rust/src/lib.rs @@ -2,85 +2,17 @@ pub mod shared { pub mod helper; include!("generated/shared.rs"); - - #[cfg(test)] - pub mod tests { - use crate::shared::ValidationFlow; - use serde_json; - - #[test] - fn test_serialize() { - let flow = ValidationFlow { - flow_id: 0, - project_id: 0, - data_types: vec![], - input_type_identifier: None, - return_type_identifier: None, - r#type: "no".to_string(), - settings: vec![], - starting_node_id: 0, - node_functions: vec![], - }; - - let str_flow = serde_json::to_string(&flow).expect("Serialization failed"); - let json_flow: serde_json::Value = - serde_json::from_str(&str_flow).expect("Failed to parse JSON"); - - let expected_json: serde_json::Value = serde_json::json!({ - "flow_id": 0, - "project_id": 0, - "type": "no", - "input_type_identifier": null, - "return_type_identifier": null, - "data_types": [], - "settings": [], - "starting_node_id": 0, - "node_functions": [] - }); - - assert_eq!(json_flow, expected_json); - } - - #[test] - fn test_deserialize() { - let json_data = r#"{ - "flow_id": 0, - "project_id": 0, - "type": "no", - "input_type_identifier": null, - "return_type_identifier": null, - "data_types": [], - "settings": [], - "starting_node_id": 0, - "node_functions": [] - }"#; - - let deserialized: Result = serde_json::from_str(json_data); - assert!(deserialized.is_ok()); - - let expected_flow = ValidationFlow { - flow_id: 0, - project_id: 0, - r#type: "no".to_string(), - settings: vec![], - data_types: vec![], - input_type_identifier: None, - return_type_identifier: None, - starting_node_id: 0, - node_functions: vec![], - }; - - assert_eq!(deserialized.unwrap(), expected_flow); - } - } + include!("generated/shared.serde.rs"); } #[cfg(feature = "aquila")] pub mod aquila { include!("generated/aquila.rs"); + include!("generated/aquila.serde.rs"); } #[cfg(feature = "sagittarius")] pub mod sagittarius { include!("generated/sagittarius.rs"); + include!("generated/sagittarius.serde.rs"); } From fd8db49dc237184fdd46b33880861d3d0c0f499b Mon Sep 17 00:00:00 2001 From: raphael-goetz Date: Sat, 1 Nov 2025 10:45:14 +0100 Subject: [PATCH 2/4] dependencies: added pbjson --- build/rust/Cargo.lock | 70 +++++++++++++++++++++++++++++++++++++++---- build/rust/Cargo.toml | 7 +++++ 2 files changed, 72 insertions(+), 5 deletions(-) diff --git a/build/rust/Cargo.lock b/build/rust/Cargo.lock index cf61a7b..738d5a7 100644 --- a/build/rust/Cargo.lock +++ b/build/rust/Cargo.lock @@ -139,6 +139,15 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" +dependencies = [ + "num-traits", +] + [[package]] name = "either" version = "1.13.0" @@ -158,7 +167,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -389,9 +398,9 @@ dependencies = [ [[package]] name = "itertools" -version = "0.13.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" dependencies = [ "either", ] @@ -464,6 +473,15 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "object" version = "0.36.7" @@ -479,6 +497,43 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "pbjson" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "898bac3fa00d0ba57a4e8289837e965baa2dee8c3749f3b11d45a64b4223d9c3" +dependencies = [ + "base64", + "serde", +] + +[[package]] +name = "pbjson-build" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af22d08a625a2213a78dbb0ffa253318c5c79ce3133d32d296655a7bdfb02095" +dependencies = [ + "heck", + "itertools", + "prost", + "prost-types", +] + +[[package]] +name = "pbjson-types" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e748e28374f10a330ee3bb9f29b828c0ac79831a32bab65015ad9b661ead526" +dependencies = [ + "bytes", + "chrono", + "pbjson", + "pbjson-build", + "prost", + "prost-build", + "serde", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -680,7 +735,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -800,7 +855,7 @@ dependencies = [ "getrandom", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -996,11 +1051,16 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" name = "tucana" version = "0.0.0" dependencies = [ + "pbjson", + "pbjson-build", + "pbjson-types", "prost", + "prost-build", "prost-types", "serde", "serde_json", "tonic", + "tonic-build", "tonic-prost", "tonic-prost-build", ] diff --git a/build/rust/Cargo.toml b/build/rust/Cargo.toml index a27969d..414cb03 100644 --- a/build/rust/Cargo.toml +++ b/build/rust/Cargo.toml @@ -15,6 +15,8 @@ prost = "0.14.0" tonic = "0.14.0" serde_json = "1.0.138" tonic-prost = "0.14.0" +pbjson = "0.8.0" +pbjson-types = "0.8.0" [lib] name = "tucana" @@ -28,3 +30,8 @@ all = ["aquila", "sagittarius"] [build-dependencies] tonic-prost-build = { version = "0.14.0" } +tonic-build = "0.14.0" +prost-build = "0.14.1" +prost = "0.14.1" +prost-types = "0.14.1" +pbjson-build = "0.8.0" From 025265a534a16713c6892eff393171e0512aff4e Mon Sep 17 00:00:00 2001 From: raphael-goetz Date: Sat, 1 Nov 2025 10:48:36 +0100 Subject: [PATCH 3/4] feat: adjusted build configuration to use pbjson instead serde --- build/rust/build.rs | 141 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 109 insertions(+), 32 deletions(-) diff --git a/build/rust/build.rs b/build/rust/build.rs index 37432c6..60b58c1 100644 --- a/build/rust/build.rs +++ b/build/rust/build.rs @@ -1,73 +1,150 @@ -use std::fs::create_dir; +use std::fs::{self, create_dir}; use std::io::Result; +use std::path::{Path, PathBuf}; fn read_proto_folders(path: &str) -> (Vec, Vec) { let mut proto_folders: Vec = Vec::new(); let mut proto_files: Vec = Vec::new(); - let proto_folder = match std::fs::read_dir(path) { - Ok(entries) => entries, - Err(error) => panic!("Cannot read the `proto` folder! Reason: {:?}", error), - }; + let proto_folder = std::fs::read_dir(path) + .unwrap_or_else(|e| panic!("Cannot read the `proto` folder! Reason: {:?}", e)); for entry in proto_folder { - let real_entry = match entry { - Ok(entry) => entry, + let entry = match entry { + Ok(e) => e, Err(_) => continue, }; - - let meta = match real_entry.metadata() { - Ok(metadata) => metadata, + let meta = match entry.metadata() { + Ok(m) => m, Err(_) => continue, }; if meta.is_dir() { - let new_path = match real_entry.path().into_os_string().into_string() { - Ok(path) => path, - Err(_) => continue, - }; + let new_path = entry + .path() + .into_os_string() + .into_string() + .unwrap_or_default(); + if new_path.is_empty() { + continue; + } proto_folders.push(new_path.clone()); - let (new_folders, new_files) = read_proto_folders(new_path.as_str()); - + let (new_folders, new_files) = read_proto_folders(&new_path); proto_folders.extend(new_folders); proto_files.extend(new_files); + } else if entry + .path() + .extension() + .map(|e| e == "proto") + .unwrap_or(false) + { + let file_name = entry.file_name().into_string().unwrap_or_default(); + proto_files.push(file_name); + } + } + + (proto_folders, proto_files) +} + +fn collect_type_fqns_from_descriptor(desc_bytes: &[u8]) -> Vec { + use prost::Message; + use prost_types::{DescriptorProto, FileDescriptorSet}; + + fn walk_msg(pkg: &str, m: &DescriptorProto, out: &mut Vec) { + let name = m.name.as_deref().unwrap_or_default(); + let fqn = if pkg.is_empty() { + format!(".{name}") } else { - let file_name = match real_entry.file_name().into_string() { - Ok(name) => name, - Err(_) => continue, + format!(".{pkg}.{name}") + }; + out.push(fqn); + for en in &m.enum_type { + let en_name = en.name.as_deref().unwrap_or_default(); + let fqn = if pkg.is_empty() { + format!(".{en_name}") + } else { + format!(".{pkg}.{en_name}") }; + out.push(fqn); + } + for nm in &m.nested_type { + walk_msg(pkg, nm, out); + } + } - proto_files.push(file_name); + let fds = FileDescriptorSet::decode(desc_bytes).expect("decode descriptor failed"); + let mut out = Vec::new(); + + for file in &fds.file { + let pkg = file.package.clone().unwrap_or_default(); + for en in &file.enum_type { + let name = en.name.clone().unwrap_or_default(); + let fqn = if pkg.is_empty() { + format!(".{name}") + } else { + format!(".{pkg}.{name}") + }; + out.push(fqn); + } + for m in &file.message_type { + walk_msg(&pkg, m, &mut out); } } - (proto_folders, proto_files) + out.sort(); + out.dedup(); + out } fn main() -> Result<()> { let path = "../../proto"; let (proto_folders, proto_files) = read_proto_folders(path); - let out_path = "src/generated"; - let serde_attribute = "#[derive(serde::Serialize, serde::Deserialize)]"; - if !std::path::Path::new(&out_path).exists() { - match create_dir(out_path) { - Err(error) => panic!("Cannot create the `generated` folder! Reason: {:?}", error), - _ => {} - }; + if !Path::new(out_path).exists() { + create_dir(out_path).expect("Cannot create the `generated` folder!"); + } + + println!("cargo:rerun-if-changed={path}"); + for f in &proto_files { + println!("cargo:rerun-if-changed={path}/{f}"); } + let descriptor_path = PathBuf::from(out_path).join("proto_descriptor.bin"); + let mut prost_cfg = prost_build::Config::new(); + prost_cfg + .file_descriptor_set_path(&descriptor_path) + .compile_well_known_types() + .extern_path(".google.protobuf", "::pbjson_types") + .protoc_arg("--experimental_allow_proto3_optional"); + + prost_cfg + .compile_protos(&proto_files, &proto_folders) + .expect("prost-build descriptor generation failed"); + let build_result = tonic_prost_build::configure() .out_dir(out_path) .build_server(true) .build_client(true) - .type_attribute(".", serde_attribute) .compile_protos(&proto_files, &proto_folders); - match build_result { - Ok(_) => Ok(()), - Err(error) => panic!("Cannot build the proto files! Reason: {:?}", error), + if let Err(error) = build_result { + panic!("Cannot build the proto files! Reason: {:?}", error); + } + + let desc_bytes = fs::read(&descriptor_path).expect("failed to read descriptor set"); + let fqns = collect_type_fqns_from_descriptor(&desc_bytes); + if fqns.is_empty() { + panic!("No message/enum types found in descriptor — pbjson can't generate code"); } + + pbjson_build::Builder::new() + .out_dir(out_path) + .register_descriptors(&desc_bytes) + .expect("failed to generate serde impl") + .build(&fqns) + .expect("pbjson-build failed"); + + Ok(()) } From 4746d72fa3bc88c23f9597134944a280ec8bafe0 Mon Sep 17 00:00:00 2001 From: raphael-goetz Date: Sat, 1 Nov 2025 10:56:29 +0100 Subject: [PATCH 4/4] docs: made the json part point to protoJSON --- build/rust/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build/rust/README.md b/build/rust/README.md index 93d0080..12cb543 100644 --- a/build/rust/README.md +++ b/build/rust/README.md @@ -9,6 +9,7 @@ The Rust Code0 gRPC library (for internal service communication) providing inter - **aquila** - gRPC services and types for Aquila (as server) component communication - **sagittarius** - gRPC services and types for Sagittarius (as server) component communication +- **shared** - shared gRPC types for the services above ## Overview @@ -43,7 +44,7 @@ let updated_flow = path::set_value("input_type_identifier", &flow_value, new_typ ### JSON Conversion for Protobuf Messages -Convert between Protobuf messages and JSON for debugging or external interfaces: +Convert between Protobuf messages and JSON for debugging or external interfaces. This uses the [protoJSON](https://protobuf.dev/programming-guides/json/) format. ```rust use tucana::shared::helper::value;