Use pbjson

build/rust/Cargo.toml

@@ -15,6 +15,8 @@ prost = "0.14.0"
 tonic = "0.14.0"
 serde_json = "1.0.138"
 tonic-prost = "0.14.0"
+pbjson = "0.8.0"
+pbjson-types = "0.8.0"
 
 [lib]
 name = "tucana"
@@ -28,3 +30,8 @@ all = ["aquila", "sagittarius"]
 
 [build-dependencies]
 tonic-prost-build = { version = "0.14.0" }
+tonic-build = "0.14.0"
+prost-build = "0.14.1"
+prost = "0.14.1"
+prost-types = "0.14.1"
+pbjson-build = "0.8.0"

build/rust/README.md

@@ -9,6 +9,7 @@ The Rust Code0 gRPC library (for internal service communication) providing inter
 
 - **aquila** - gRPC services and types for Aquila (as server) component communication
 - **sagittarius** - gRPC services and types for Sagittarius (as server) component communication
+- **shared** - shared gRPC types for the services above
 
 ## Overview
 
@@ -43,7 +44,7 @@ let updated_flow = path::set_value("input_type_identifier", &flow_value, new_typ
 
 ### JSON Conversion for Protobuf Messages
 
-Convert between Protobuf messages and JSON for debugging or external interfaces:
+Convert between Protobuf messages and JSON for debugging or external interfaces. This uses the [protoJSON](https://protobuf.dev/programming-guides/json/) format.
 
 ```rust
 use tucana::shared::helper::value;

build/rust/build.rs

@@ -1,73 +1,150 @@
-use std::fs::create_dir;
+use std::fs::{self, create_dir};
 use std::io::Result;
+use std::path::{Path, PathBuf};
 
 fn read_proto_folders(path: &str) -> (Vec<String>, Vec<String>) {
     let mut proto_folders: Vec<String> = Vec::new();
     let mut proto_files: Vec<String> = Vec::new();
 
-    let proto_folder = match std::fs::read_dir(path) {
-        Ok(entries) => entries,
-        Err(error) => panic!("Cannot read the `proto` folder! Reason: {:?}", error),
-    };
+    let proto_folder = std::fs::read_dir(path)
+        .unwrap_or_else(|e| panic!("Cannot read the `proto` folder! Reason: {:?}", e));
 
     for entry in proto_folder {
-        let real_entry = match entry {
-            Ok(entry) => entry,
+        let entry = match entry {
+            Ok(e) => e,
             Err(_) => continue,
         };
-
-        let meta = match real_entry.metadata() {
-            Ok(metadata) => metadata,
+        let meta = match entry.metadata() {
+            Ok(m) => m,
             Err(_) => continue,
         };
 
         if meta.is_dir() {
-            let new_path = match real_entry.path().into_os_string().into_string() {
-                Ok(path) => path,
-                Err(_) => continue,
-            };
+            let new_path = entry
+                .path()
+                .into_os_string()
+                .into_string()
+                .unwrap_or_default();
+            if new_path.is_empty() {
+                continue;
+            }
 
             proto_folders.push(new_path.clone());
-            let (new_folders, new_files) = read_proto_folders(new_path.as_str());
-
+            let (new_folders, new_files) = read_proto_folders(&new_path);
             proto_folders.extend(new_folders);
             proto_files.extend(new_files);
+        } else if entry
+            .path()
+            .extension()
+            .map(|e| e == "proto")
+            .unwrap_or(false)
+        {
+            let file_name = entry.file_name().into_string().unwrap_or_default();
+            proto_files.push(file_name);
+        }
+    }
+
+    (proto_folders, proto_files)
+}
+
+fn collect_type_fqns_from_descriptor(desc_bytes: &[u8]) -> Vec<String> {
+    use prost::Message;
+    use prost_types::{DescriptorProto, FileDescriptorSet};
+
+    fn walk_msg(pkg: &str, m: &DescriptorProto, out: &mut Vec<String>) {
+        let name = m.name.as_deref().unwrap_or_default();
+        let fqn = if pkg.is_empty() {
+            format!(".{name}")
         } else {
-            let file_name = match real_entry.file_name().into_string() {
-                Ok(name) => name,
-                Err(_) => continue,
+            format!(".{pkg}.{name}")
+        };
+        out.push(fqn);
+        for en in &m.enum_type {
+            let en_name = en.name.as_deref().unwrap_or_default();
+            let fqn = if pkg.is_empty() {
+                format!(".{en_name}")
+            } else {
+                format!(".{pkg}.{en_name}")
             };
+            out.push(fqn);
+        }
+        for nm in &m.nested_type {
+            walk_msg(pkg, nm, out);
+        }
+    }
 
-            proto_files.push(file_name);
+    let fds = FileDescriptorSet::decode(desc_bytes).expect("decode descriptor failed");
+    let mut out = Vec::new();
+
+    for file in &fds.file {
+        let pkg = file.package.clone().unwrap_or_default();
+        for en in &file.enum_type {
+            let name = en.name.clone().unwrap_or_default();
+            let fqn = if pkg.is_empty() {
+                format!(".{name}")
+            } else {
+                format!(".{pkg}.{name}")
+            };
+            out.push(fqn);
+        }
+        for m in &file.message_type {
+            walk_msg(&pkg, m, &mut out);
         }
     }
 
-    (proto_folders, proto_files)
+    out.sort();
+    out.dedup();
+    out
 }
 
 fn main() -> Result<()> {
     let path = "../../proto";
     let (proto_folders, proto_files) = read_proto_folders(path);
-
     let out_path = "src/generated";
-    let serde_attribute = "#[derive(serde::Serialize, serde::Deserialize)]";
 
-    if !std::path::Path::new(&out_path).exists() {
-        match create_dir(out_path) {
-            Err(error) => panic!("Cannot create the `generated` folder! Reason: {:?}", error),
-            _ => {}
-        };
+    if !Path::new(out_path).exists() {
+        create_dir(out_path).expect("Cannot create the `generated` folder!");
+    }
+
+    println!("cargo:rerun-if-changed={path}");
+    for f in &proto_files {
+        println!("cargo:rerun-if-changed={path}/{f}");
     }
 
+    let descriptor_path = PathBuf::from(out_path).join("proto_descriptor.bin");
+    let mut prost_cfg = prost_build::Config::new();
+    prost_cfg
+        .file_descriptor_set_path(&descriptor_path)
+        .compile_well_known_types()
+        .extern_path(".google.protobuf", "::pbjson_types")
+        .protoc_arg("--experimental_allow_proto3_optional");
+
+    prost_cfg
+        .compile_protos(&proto_files, &proto_folders)
+        .expect("prost-build descriptor generation failed");
+
     let build_result = tonic_prost_build::configure()
         .out_dir(out_path)
         .build_server(true)
         .build_client(true)
-        .type_attribute(".", serde_attribute)
         .compile_protos(&proto_files, &proto_folders);
 
-    match build_result {
-        Ok(_) => Ok(()),
-        Err(error) => panic!("Cannot build the proto files! Reason: {:?}", error),
+    if let Err(error) = build_result {
+        panic!("Cannot build the proto files! Reason: {:?}", error);
+    }
+
+    let desc_bytes = fs::read(&descriptor_path).expect("failed to read descriptor set");
+    let fqns = collect_type_fqns_from_descriptor(&desc_bytes);
+    if fqns.is_empty() {
+        panic!("No message/enum types found in descriptor — pbjson can't generate code");
     }
+
+    pbjson_build::Builder::new()
+        .out_dir(out_path)
+        .register_descriptors(&desc_bytes)
+        .expect("failed to generate serde impl")
+        .build(&fqns)
+        .expect("pbjson-build failed");
+
+    Ok(())
 }