From 0ba8b1e0e0aa6ce82cb96de5f933e46d3bb946ac Mon Sep 17 00:00:00 2001 From: Johann Tuffe Date: Mon, 6 Feb 2017 11:13:20 +0800 Subject: [PATCH 1/4] parse nested messages --- codegen/src/parser.rs | 117 ++++++++++++++++++++++++------------------ codegen/src/types.rs | 3 +- 2 files changed, 70 insertions(+), 50 deletions(-) diff --git a/codegen/src/parser.rs b/codegen/src/parser.rs index 24da7f2b..e1af35ca 100644 --- a/codegen/src/parser.rs +++ b/codegen/src/parser.rs @@ -49,22 +49,12 @@ named!(reserved_names>, many0!(alt!(br | tag!(",") => { |_| () })) >> (name))) >> (names) )); - - -named!(default_value, - do_parse!(tag!("[") >> many0!(br) >> tag!("default") >> many0!(br) >> tag!("=") >> many0!(br) >> - default: word >> many0!(br) >> tag!("]") >> - (default) )); -named!(deprecated, - do_parse!(tag!("[") >> many0!(br) >> tag!("deprecated") >> many0!(br) >> tag!("=") >> many0!(br) >> - deprecated: map_res!(word_ref, str::FromStr::from_str) >> many0!(br) >> tag!("]") >> - (deprecated) )); - -named!(packed, - do_parse!(tag!("[") >> many0!(br) >> tag!("packed") >> many0!(br) >> tag!("=") >> many0!(br) >> - packed: map_res!(word_ref, str::FromStr::from_str) >> many0!(br) >> tag!("]") >> - (packed) )); +named!(key_val<(&str, &str)>, + do_parse!(tag!("[") >> many0!(br) >> + key: word_ref >> many0!(br) >> tag!("=") >> many0!(br) >> + value: word_ref >> many0!(br) >> tag!("]") >> many0!(br) >> + ((key, value)) )); named!(frequency, alt!(tag!("optional") => { |_| Frequency::Optional } | @@ -74,38 +64,68 @@ named!(frequency, named!(message_field, do_parse!(frequency: opt!(frequency) >> many1!(br) >> typ: word >> many1!(br) >> - name: word >> many0!(br) >> - tag!("=") >> many0!(br) >> + name: word >> many0!(br) >> tag!("=") >> many0!(br) >> number: map_res!(map_res!(digit, str::from_utf8), str::FromStr::from_str) >> many0!(br) >> - default: opt!(default_value) >> many0!(br) >> - deprecated: opt!(deprecated) >> many0!(br) >> - packed: opt!(packed) >> many0!(br) >> tag!(";") >> many0!(br) >> + key_vals: many0!(key_val) >> tag!(";") >> (Field { name: name, frequency: frequency.unwrap_or(Frequency::Optional), typ: typ, number: number, - default: default, - packed: packed, + default: key_vals.iter().find(|&&(k, _)| k == "default") + .map(|&(_, v)| v.to_string()), + packed: key_vals.iter().find(|&&(k, _)| k == "packed") + .map(|&(_, v)| str::FromStr::from_str(v) + .expect("Cannot parse Packed value")), boxed: false, - deprecated: deprecated.unwrap_or(false), + deprecated: key_vals.iter().find(|&&(k, _)| k == "deprecated") + .map_or(false, |&(_, v)| str::FromStr::from_str(v) + .expect("Cannot parse Deprecated value")), }) )); -named!(message, +enum MessageEvent { + Messages(Vec), + Field(Field), + ReservedNums(Vec), + ReservedNames(Vec), + Ignore, +} + +named!(message_event, alt!(reserved_nums => { |r| MessageEvent::ReservedNums(r) } | + reserved_names => { |r| MessageEvent::ReservedNames(r) } | + message_field => { |f| MessageEvent::Field(f) } | + message => { |m| MessageEvent::Messages(m) } | + br => { |_| MessageEvent::Ignore })); + +named!(message_events<(String, Vec)>, do_parse!(tag!("message") >> many0!(br) >> name: word >> many0!(br) >> tag!("{") >> many0!(br) >> - reserved_nums: opt!(reserved_nums) >> many0!(br) >> - reserved_names: opt!(reserved_names) >> many0!(br) >> - fields: many0!(message_field) >> - tag!("}") >> - (Message { - name: name, - fields: fields, - reserved_nums: reserved_nums, - reserved_names: reserved_names, - imported: false, - }) )); + events: many0!(message_event) >> + many0!(br) >> tag!("}") >> + ((name, events)) )); + +named!(message>, + map!(message_events, |(name, events): (String, Vec)| { + let mut messages = Vec::new(); + let mut msg = Message { name: name.clone(), .. Message::default() }; + for e in events { + match e { + MessageEvent::Field(f) => msg.fields.push(f), + MessageEvent::ReservedNums(r) => msg.reserved_nums = Some(r), + MessageEvent::ReservedNames(r) => msg.reserved_names = Some(r), + MessageEvent::Messages(ms) => { + for mut m in ms { + m.parents.push(name.clone()); + messages.push(m) + } + }, + MessageEvent::Ignore => (), + } + } + messages.push(msg); + messages + })); named!(enum_field<(String, i32)>, do_parse!(name: word >> many0!(br) >> @@ -123,9 +143,8 @@ named!(enumerator, imported: false, }))); -named!(ignore<()>, - do_parse!(alt!(tag!("package") | tag!("option")) >> many1!(br) >> - take_until_and_consume!(";") >> ())); +named!(option_ignore<()>, + do_parse!(tag!("option") >> many1!(br) >> take_until_and_consume!(";") >> ())); named!(service_ignore<()>, do_parse!(tag!("service") >> many1!(br) >> word >> many0!(br) >> tag!("{") >> @@ -135,7 +154,7 @@ enum Event { Syntax(Syntax), Import(PathBuf), Package(String), - Message(Message), + Messages(Vec), Enum(Enumerator), Ignore } @@ -144,9 +163,9 @@ named!(event, alt!(syntax => { |s| Event::Syntax(s) } | import => { |i| Event::Import(i) } | package => { |p| Event::Package(p) } | - message => { |m| Event::Message(m) } | + message => { |m| Event::Messages(m) } | enumerator => { |e| Event::Enum(e) } | - ignore => { |_| Event::Ignore } | + option_ignore => { |_| Event::Ignore } | service_ignore => { |_| Event::Ignore } | br => { |_| Event::Ignore })); @@ -158,7 +177,7 @@ named!(pub file_descriptor, Event::Syntax(s) => desc.syntax = s, Event::Import(i) => desc.import_paths.push(i), Event::Package(p) => desc.package = p.split('.').map(|s| s.to_string()).collect(), - Event::Message(m) => desc.messages.push(m), + Event::Messages(m) => desc.messages.extend(m), Event::Enum(e) => desc.enums.push(e), Event::Ignore => (), } @@ -168,6 +187,8 @@ named!(pub file_descriptor, #[cfg(test)] mod test { + use super::*; + #[test] fn test_message() { let msg = r#"message ReferenceData @@ -186,7 +207,8 @@ mod test { let mess = message(msg.as_bytes()); if let ::nom::IResult::Done(_, mess) = mess { - assert_eq!(10, mess.fields.len()); + assert!(mess.len() == 1); + assert_eq!(10, mess[0].fields.len()); } } @@ -207,12 +229,9 @@ mod test { #[test] fn test_ignore() { - let msg = r#"package com.test.v0; - - option optimize_for = SPEED; - "#; + let msg = r#"option optimize_for = SPEED;"#; - match ignore(msg.as_bytes()) { + match option_ignore(msg.as_bytes()) { ::nom::IResult::Done(_, _) => (), e => panic!("Expecting done {:?}", e), } @@ -244,6 +263,6 @@ mod test { } "#; let desc = file_descriptor(msg.as_bytes()).to_full_result().unwrap(); - assert_eq!(Some("foo.bar".to_string()), desc.package); + assert_eq!(vec!("foo".to_string(), "bar".to_string()), desc.package); } } diff --git a/codegen/src/types.rs b/codegen/src/types.rs index 1bb4facf..e0cbe81e 100644 --- a/codegen/src/types.rs +++ b/codegen/src/types.rs @@ -410,8 +410,9 @@ impl Field { } } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] pub struct Message { + pub parents: Vec, pub name: String, pub fields: Vec, pub reserved_nums: Option>, From bc955b62a933510940220ff26baf42edc129feff Mon Sep 17 00:00:00 2001 From: Johann Tuffe Date: Mon, 6 Feb 2017 13:56:07 +0800 Subject: [PATCH 2/4] set parent message into package --- codegen/src/parser.rs | 48 ++++++++++++--------- codegen/src/types.rs | 98 +++++++++++++++++++++++++------------------ 2 files changed, 87 insertions(+), 59 deletions(-) diff --git a/codegen/src/parser.rs b/codegen/src/parser.rs index e1af35ca..15ce608d 100644 --- a/codegen/src/parser.rs +++ b/codegen/src/parser.rs @@ -84,7 +84,7 @@ named!(message_field, }) )); enum MessageEvent { - Messages(Vec), + Message(Message), Field(Field), ReservedNums(Vec), ReservedNames(Vec), @@ -94,7 +94,7 @@ enum MessageEvent { named!(message_event, alt!(reserved_nums => { |r| MessageEvent::ReservedNums(r) } | reserved_names => { |r| MessageEvent::ReservedNames(r) } | message_field => { |f| MessageEvent::Field(f) } | - message => { |m| MessageEvent::Messages(m) } | + message => { |m| MessageEvent::Message(m) } | br => { |_| MessageEvent::Ignore })); named!(message_events<(String, Vec)>, @@ -105,26 +105,19 @@ named!(message_events<(String, Vec)>, many0!(br) >> tag!("}") >> ((name, events)) )); -named!(message>, +named!(message, map!(message_events, |(name, events): (String, Vec)| { - let mut messages = Vec::new(); let mut msg = Message { name: name.clone(), .. Message::default() }; for e in events { match e { MessageEvent::Field(f) => msg.fields.push(f), MessageEvent::ReservedNums(r) => msg.reserved_nums = Some(r), MessageEvent::ReservedNames(r) => msg.reserved_names = Some(r), - MessageEvent::Messages(ms) => { - for mut m in ms { - m.parents.push(name.clone()); - messages.push(m) - } - }, + MessageEvent::Message(m) => msg.messages.push(m), MessageEvent::Ignore => (), } } - messages.push(msg); - messages + msg })); named!(enum_field<(String, i32)>, @@ -141,6 +134,7 @@ named!(enumerator, name: name, fields: fields, imported: false, + package: "".to_string(), }))); named!(option_ignore<()>, @@ -154,7 +148,7 @@ enum Event { Syntax(Syntax), Import(PathBuf), Package(String), - Messages(Vec), + Message(Message), Enum(Enumerator), Ignore } @@ -163,7 +157,7 @@ named!(event, alt!(syntax => { |s| Event::Syntax(s) } | import => { |i| Event::Import(i) } | package => { |p| Event::Package(p) } | - message => { |m| Event::Messages(m) } | + message => { |m| Event::Message(m) } | enumerator => { |e| Event::Enum(e) } | option_ignore => { |_| Event::Ignore } | service_ignore => { |_| Event::Ignore } | @@ -176,8 +170,8 @@ named!(pub file_descriptor, match event { Event::Syntax(s) => desc.syntax = s, Event::Import(i) => desc.import_paths.push(i), - Event::Package(p) => desc.package = p.split('.').map(|s| s.to_string()).collect(), - Event::Messages(m) => desc.messages.extend(m), + Event::Package(p) => desc.package = p, + Event::Message(m) => desc.messages.push(m), Event::Enum(e) => desc.enums.push(e), Event::Ignore => (), } @@ -207,8 +201,7 @@ mod test { let mess = message(msg.as_bytes()); if let ::nom::IResult::Done(_, mess) = mess { - assert!(mess.len() == 1); - assert_eq!(10, mess[0].fields.len()); + assert_eq!(10, mess.fields.len()); } } @@ -263,6 +256,23 @@ mod test { } "#; let desc = file_descriptor(msg.as_bytes()).to_full_result().unwrap(); - assert_eq!(vec!("foo".to_string(), "bar".to_string()), desc.package); + assert_eq!("foo.bar".to_string(), desc.package); + } + + #[test] + fn test_nested_message() { + let msg = r#"message A + { + message B { + repeated int32 a = 1; + optional string b = 2; + } + optional b = 1; + }"#; + + let mess = message(msg.as_bytes()); + if let ::nom::IResult::Done(_, mess) = mess { + assert!(mess.messages.len() == 1); + } } } diff --git a/codegen/src/types.rs b/codegen/src/types.rs index e0cbe81e..0b6d4ce9 100644 --- a/codegen/src/types.rs +++ b/codegen/src/types.rs @@ -72,7 +72,7 @@ impl Field { } fn is_message(&self, msgs: &[Message]) -> bool { - msgs.iter().any(|m| m.name == self.typ) + self.find_message(msgs).is_some() } fn is_enum(&self, msgs: &[Message]) -> bool { @@ -93,6 +93,22 @@ impl Field { } } + fn find_message<'a, 'b>(&'a self, msgs: &'b [Message]) -> Option<&'b Message> { + if self.name.contains('.') { + msgs.iter().filter(|m| m.package.is_empty()) + .find(|m| m.name == self.typ) + + } else { + msgs.iter().filter(|m| m.package.is_empty()) +// .chain(msgs.messages.iter()) + .find(|m| m.name == self.typ) + } + } + + fn find_enum<'a, 'b>(&'a self, enums: &'b [Enumerator]) -> Option<&'b Enumerator> { + enums.iter().find(|m| m.name == self.typ) + } + fn has_unregular_default(&self, enums: &[Enumerator], msgs: &[Message]) -> bool { match self.default { None => false, @@ -101,23 +117,14 @@ impl Field { "bool" => *d != "false", "Cow<'a, str>" => *d != "\"\"", "Cow<'a, [u8]>" => *d != "[]", - t => match enums.iter().find(|e| e.name == self.typ) { - Some(e) => t != e.fields[0].0, - None => false, // Messages are regular defaults - } + t => self.find_enum(enums).map_or(false, |e| t != e.fields[0].0), } } } fn has_lifetime(&self, msgs: &[Message]) -> bool { - // borrow bytes and string if self.is_cow() { return true; } - - // borrow messages that have lifetime (ie they have at least one borrowed field) - match msgs.iter().find(|m| m.name == self.typ) { - Some(ref m) if m.has_lifetime(msgs) => true, - _ => false, - } + self.find_message(msgs).map_or(false, |m| m.has_lifetime(msgs)) } fn rust_type(&self, msgs: &[Message]) -> String { @@ -130,7 +137,7 @@ impl Field { "double" => "f64".to_string(), "string" => "Cow<'a, str>".to_string(), "bytes" => "Cow<'a, [u8]>".to_string(), - t => msgs.iter().find(|m| m.name == t).map_or(t.to_string(), |m| if m.has_lifetime(msgs) { + t => self.find_message(msgs).map_or(t.to_string(), |m| if m.has_lifetime(msgs) { format!("{}<'a>", t.replace(".", "::")) } else { t.replace(".", "::") @@ -153,7 +160,7 @@ impl Field { "fixed64" | "sfixed64" | "double" => 1, "fixed32" | "sfixed32" | "float" => 5, "string" | "bytes" => 2, - t => if msgs.iter().any(|m| m.name == t) { 2 } else { 0 /* enum */ } + _ => if self.is_message(msgs) { 2 } else { 0 /* enum */ } } } @@ -412,12 +419,13 @@ impl Field { #[derive(Debug, Clone, Default)] pub struct Message { - pub parents: Vec, + pub messages: Vec, pub name: String, pub fields: Vec, pub reserved_nums: Option>, pub reserved_names: Option>, pub imported: bool, + pub package: String, // package from imports + nested classes } impl Message { @@ -549,6 +557,26 @@ impl Message { } Ok(()) } + + fn set_package(&mut self, package: &str) { + if package.is_empty() { + for m in &mut self.messages { + // set package = name to nested messages + m.set_package(&self.name); + } + } else { + if !self.package.is_empty() { + self.package.push('.'); + } + self.package.push_str(package); + let package = format!("{}.{}", self.package, self.name); + for m in &mut self.messages { + // set package = package.name to nested messages + m.set_package(&package); + } + } + } + } #[derive(Debug, Clone)] @@ -556,6 +584,7 @@ pub struct Enumerator { pub name: String, pub fields: Vec<(String, i32)>, pub imported: bool, + pub package: String, } impl Enumerator { @@ -597,7 +626,7 @@ impl Enumerator { #[derive(Debug, Default)] pub struct FileDescriptor { pub import_paths: Vec, - pub package: Vec, + pub package: String, pub syntax: Syntax, pub messages: Vec, pub enums: Vec, @@ -654,28 +683,17 @@ impl FileDescriptor { f.fetch_imports(&import_path)?; // if the proto has a packge then the names will be prefixed - if f.package.is_empty() { - self.messages.extend(f.messages.drain(..).map(|mut m| { - m.imported = true; - m - })); - self.enums.extend(f.enums.drain(..).map(|mut e| { - e.imported = true; - e - })); - } else { - let name_prefix = f.package.join("."); - self.messages.extend(f.messages.drain(..).map(|mut m| { - m.name = format!("{}.{}", name_prefix, m.name); - m.imported = true; - m - })); - self.enums.extend(f.enums.drain(..).map(|mut e| { - e.name = format!("{}.{}", name_prefix, e.name); - e.imported = true; - e - })); - } + let package = f.package.clone(); + self.messages.extend(f.messages.drain(..).map(|mut m| { + m.set_package(&package); + m.imported = true; + m + })); + self.enums.extend(f.enums.drain(..).map(|mut e| { + e.package = package.clone(); + e.imported = true; + e + })); } Ok(()) } @@ -791,7 +809,7 @@ impl FileDescriptor { } fn write_package_start(&self, w: &mut W) -> Result<()> { - for p in &self.package { writeln!(w, "pub mod {} {{", p)?; } + for p in self.package.split('.') { writeln!(w, "pub mod {} {{", p)?; } if !self.package.is_empty() { writeln!(w, "")?; } Ok(()) } @@ -799,7 +817,7 @@ impl FileDescriptor { fn write_package_end(&self, w: &mut W) -> Result<()> { if self.package.is_empty() { return Ok(()); } writeln!(w, "")?; - for _ in &self.package { writeln!(w, "}}")?; } + for _ in self.package.split('.') { writeln!(w, "}}")?; } Ok(()) } From 828ab295ff8e81b9538f66a37168cffe755dc79b Mon Sep 17 00:00:00 2001 From: Johann Tuffe Date: Mon, 6 Feb 2017 17:19:10 +0800 Subject: [PATCH 3/4] add mod_Message nodule for each nested level --- codegen/src/types.rs | 186 ++++++++++++++++---------- examples/codegen/data_types.proto | 9 ++ examples/codegen/data_types.rs | 74 +++++++++- examples/codegen/data_types_import.rs | 4 +- examples/codegen_example.rs | 2 +- 5 files changed, 201 insertions(+), 74 deletions(-) diff --git a/codegen/src/types.rs b/codegen/src/types.rs index 0b6d4ce9..7581f8d7 100644 --- a/codegen/src/types.rs +++ b/codegen/src/types.rs @@ -63,11 +63,17 @@ impl Field { } /// searches if the message must be boxed - fn is_leaf(&self, leaf_messages: &[&str], msgs: &[Message]) -> bool { + fn is_leaf(&self, leaf_messages: &[String], msgs: &[Message]) -> bool { match self.frequency { Frequency::Repeated | Frequency::Required => return true, Frequency::Optional if !self.is_message(msgs) => true, - _ => leaf_messages.iter().any(|m| m == &self.typ), + _ => { + let typ = match self.typ.rfind('.') { + Some(p) => &self.typ[p + 1..], + None => &self.typ[..], + }; + leaf_messages.iter().any(|m| &*m == &typ) + }, } } @@ -94,15 +100,25 @@ impl Field { } fn find_message<'a, 'b>(&'a self, msgs: &'b [Message]) -> Option<&'b Message> { - if self.name.contains('.') { - msgs.iter().filter(|m| m.package.is_empty()) - .find(|m| m.name == self.typ) - } else { - msgs.iter().filter(|m| m.package.is_empty()) -// .chain(msgs.messages.iter()) - .find(|m| m.name == self.typ) + let mut found = match self.typ.rfind('.') { + Some(p) => { + let package = &self.typ[..p]; + let name = &self.typ[(p + 1)..]; + msgs.iter().find(|m| m.package == package && m.name == name) + }, + None => msgs.iter().find(|m| m.name == self.typ), + }; + + if found.is_none() { + // recursively search into nested messages + for m in msgs { + found = self.find_message(&m.messages); + if found.is_some() { break; } + } } + + found } fn find_enum<'a, 'b>(&'a self, enums: &'b [Enumerator]) -> Option<&'b Enumerator> { @@ -137,11 +153,15 @@ impl Field { "double" => "f64".to_string(), "string" => "Cow<'a, str>".to_string(), "bytes" => "Cow<'a, [u8]>".to_string(), - t => self.find_message(msgs).map_or(t.to_string(), |m| if m.has_lifetime(msgs) { - format!("{}<'a>", t.replace(".", "::")) - } else { - t.replace(".", "::") - }) + t => match self.find_message(msgs) { + Some(m) => { + let lifetime = if m.has_lifetime(msgs) { "<'a>" } else { "" }; + let package = m.package.split('.').filter(|p| !p.is_empty()) + .map(|p| format!("mod_{}::", p)).collect::(); + format!("{}{}{}", package, m.name, lifetime) + }, + None => t.replace(".", "::"), // enum + } } } @@ -175,10 +195,17 @@ impl Field { } fn read_fn(&self, msgs: &[Message]) -> String { - if self.is_message(msgs) { - format!("read_message(bytes, {}::from_reader)", self.typ.replace(".", "::")) - } else { - format!("read_{}(bytes)", self.get_type(msgs)) + match self.find_message(msgs) { + Some(m) if m.package.is_empty()=> { + format!("read_message(bytes, {}::from_reader)", m.name) + }, + Some(m) => { + format!("read_message(bytes, {}{}::from_reader)", + m.package.split('.').map(|p| format!("mod_{}::", p)).collect::(), m.name) + } + None => { + format!("read_{}(bytes)", self.get_type(msgs)) + } } } @@ -430,8 +457,8 @@ pub struct Message { impl Message { - fn is_leaf(&self, leaf_messages: &[&str], msgs: &[Message]) -> bool { - self.fields.iter().all(|f| f.is_leaf(leaf_messages, msgs) || f.deprecated) + fn is_leaf(&self, leaf_messages: &[String], msgs: &[Message]) -> bool { + self.imported || self.fields.iter().all(|f| f.is_leaf(leaf_messages, msgs) || f.deprecated) } fn has_lifetime(&self, msgs: &[Message]) -> bool { @@ -559,21 +586,13 @@ impl Message { } fn set_package(&mut self, package: &str) { + // set package = current_package.package.name to nested messages if package.is_empty() { - for m in &mut self.messages { - // set package = name to nested messages - m.set_package(&self.name); - } + for m in &mut self.messages { m.set_package(&self.name); } } else { - if !self.package.is_empty() { - self.package.push('.'); - } - self.package.push_str(package); - let package = format!("{}.{}", self.package, self.name); - for m in &mut self.messages { - // set package = package.name to nested messages - m.set_package(&package); - } + self.package = package.to_string(); + let child_package = format!("{}.{}", package, self.name); + for m in &mut self.messages { m.set_package(&child_package); } } } @@ -637,7 +656,10 @@ impl FileDescriptor { pub fn write_proto>(in_file: P, out_file: P) -> Result<()> { let mut desc = FileDescriptor::read_proto(&in_file)?; desc.fetch_imports(in_file.as_ref())?; - desc.break_cycles(); + + let mut leaf_messages = Vec::new(); + break_cycles(&mut desc.messages, &mut leaf_messages); + desc.sanity_checks(in_file.as_ref())?; desc.set_defaults(); @@ -677,6 +699,9 @@ impl FileDescriptor { /// Get messages and enums from imports fn fetch_imports(&mut self, in_file: &Path) -> Result<()> { + for m in &mut self.messages { + m.set_package(""); + } for p in &self.import_paths { let import_path = get_imported_path(&in_file, p); let mut f = FileDescriptor::read_proto(&import_path)?; @@ -698,39 +723,6 @@ impl FileDescriptor { Ok(()) } - fn break_cycles(&mut self) { - - let message_names = self.messages.iter().map(|m| m.name.to_string()).collect::>(); - - let mut leaf_messages = Vec::new(); - let mut undef_messages = (0..self.messages.len()).collect::>(); - while !undef_messages.is_empty() { - let len = undef_messages.len(); - let mut new_undefs = Vec::new(); - for i in undef_messages.into_iter() { - if self.messages[i].is_leaf(&leaf_messages, &self.messages) { - leaf_messages.push(&*message_names[i]) - } else { - new_undefs.push(i); - } - } - undef_messages = new_undefs; - if len == undef_messages.len() { - // try boxing messages, one by one ... - let k = undef_messages.pop().unwrap(); - { - let mut m = self.messages[k].clone(); - for f in m.fields.iter_mut() { - if !f.is_leaf(&leaf_messages, &self.messages) { - f.boxed = true; - } - } - self.messages[k] = m; - } - } - } - } - fn set_defaults(&mut self) { // if proto3, then changes several defaults if let Syntax::Proto3 = self.syntax { @@ -809,8 +801,9 @@ impl FileDescriptor { } fn write_package_start(&self, w: &mut W) -> Result<()> { - for p in self.package.split('.') { writeln!(w, "pub mod {} {{", p)?; } - if !self.package.is_empty() { writeln!(w, "")?; } + if self.package.is_empty() { return Ok(()); } + for p in self.package.split('.') { writeln!(w, "pub mod mod_{} {{", p)?; } + writeln!(w, "")?; Ok(()) } @@ -843,6 +836,24 @@ impl FileDescriptor { m.write_impl_message_read(w, &self.enums, &self.messages)?; writeln!(w, "")?; m.write_impl_message_write(w, &self.messages)?; + + if !m.messages.is_empty() { + writeln!(w, "")?; + writeln!(w, "pub mod mod_{} {{", m.name)?; + writeln!(w, "")?; + writeln!(w, "use super::*;")?; + for m_sub in &m.messages { + println!("Writing message mod_{}::{}", m.name, m_sub.name); + writeln!(w, "")?; + m_sub.write_definition(w, &self.enums, &self.messages)?; + writeln!(w, "")?; + m_sub.write_impl_message_read(w, &self.enums, &self.messages)?; + writeln!(w, "")?; + m_sub.write_impl_message_write(w, &self.messages)?; + } + writeln!(w, "")?; + writeln!(w, "}}")?; + } } Ok(()) } @@ -851,3 +862,40 @@ impl FileDescriptor { fn get_imported_path, Q: AsRef>(in_file: P, import: Q) -> PathBuf { in_file.as_ref().parent().map_or_else(|| import.as_ref().into(), |p| p.join(import.as_ref())) } + +fn break_cycles(messages: &mut [Message], leaf_messages: &mut Vec) { + + for m in messages.iter_mut() { + break_cycles(&mut m.messages, leaf_messages); + } + + let message_names = messages.iter().map(|m| m.name.to_string()).collect::>(); + + let mut undef_messages = (0..messages.len()).collect::>(); + while !undef_messages.is_empty() { + let len = undef_messages.len(); + let mut new_undefs = Vec::new(); + for i in undef_messages { + if messages[i].is_leaf(&leaf_messages, &messages) { + leaf_messages.push(message_names[i].clone()) + } else { + new_undefs.push(i); + } + } + undef_messages = new_undefs; + if len == undef_messages.len() { + // try boxing messages, one by one ... + let k = undef_messages.pop().unwrap(); + { + let mut m = messages[k].clone(); + for f in m.fields.iter_mut() { + if !f.is_leaf(&leaf_messages, &messages) { + f.boxed = true; + } + } + messages[k] = m; + } + } + } +} + diff --git a/examples/codegen/data_types.proto b/examples/codegen/data_types.proto index cb4a42e7..c8467c71 100644 --- a/examples/codegen/data_types.proto +++ b/examples/codegen/data_types.proto @@ -31,4 +31,13 @@ message FooMessage { repeated int32 f_repeated_int32 = 19; repeated int32 f_repeated_packed_int32 = 20 [ packed = true ]; optional a.b.ImportedMessage f_imported = 21; + optional BazMessage.Nested f_nested = 22; } + +message BazMessage { + message Nested { + required int32 f_nested = 1; + } + optional Nested nested = 1; +} + diff --git a/examples/codegen/data_types.rs b/examples/codegen/data_types.rs index c7a79bac..398f7478 100644 --- a/examples/codegen/data_types.rs +++ b/examples/codegen/data_types.rs @@ -85,7 +85,8 @@ pub struct FooMessage<'a> { pub f_bar_message: Option, pub f_repeated_int32: Vec, pub f_repeated_packed_int32: Vec, - pub f_imported: Option, + pub f_imported: Option, + pub f_nested: Option, } impl<'a> FooMessage<'a> { @@ -113,7 +114,8 @@ impl<'a> FooMessage<'a> { Ok(146) => msg.f_bar_message = Some(r.read_message(bytes, BarMessage::from_reader)?), Ok(152) => msg.f_repeated_int32.push(r.read_int32(bytes)?), Ok(162) => msg.f_repeated_packed_int32 = r.read_packed(bytes, |r, bytes| r.read_int32(bytes))?, - Ok(170) => msg.f_imported = Some(r.read_message(bytes, a::b::ImportedMessage::from_reader)?), + Ok(170) => msg.f_imported = Some(r.read_message(bytes, mod_a::mod_b::ImportedMessage::from_reader)?), + Ok(178) => msg.f_nested = Some(r.read_message(bytes, mod_BazMessage::Nested::from_reader)?), Ok(t) => { r.read_unknown(bytes, t)?; } Err(e) => return Err(e), } @@ -145,6 +147,7 @@ impl<'a> MessageWrite for FooMessage<'a> { + self.f_repeated_int32.iter().map(|s| 2 + sizeof_int32(*s)).sum::() + if self.f_repeated_packed_int32.is_empty() { 0 } else { 2 + sizeof_var_length(self.f_repeated_packed_int32.iter().map(|s| sizeof_int32(*s)).sum::()) } + self.f_imported.as_ref().map_or(0, |m| 2 + sizeof_var_length(m.get_size())) + + self.f_nested.as_ref().map_or(0, |m| 2 + sizeof_var_length(m.get_size())) } fn write_message(&self, r: &mut Writer) -> Result<()> { @@ -169,6 +172,73 @@ impl<'a> MessageWrite for FooMessage<'a> { for s in &self.f_repeated_int32 { r.write_int32_with_tag(152, *s)? } r.write_packed_repeated_field_with_tag(162, &self.f_repeated_packed_int32, |r, m| r.write_int32(*m), &|m| sizeof_int32(*m))?; if let Some(ref s) = self.f_imported { r.write_message_with_tag(170, s)?; } + if let Some(ref s) = self.f_nested { r.write_message_with_tag(178, s)?; } Ok(()) } } + +#[derive(Debug, Default, PartialEq, Clone)] +pub struct BazMessage { + pub nested: Option, +} + +impl BazMessage { + pub fn from_reader(r: &mut BytesReader, bytes: &[u8]) -> Result { + let mut msg = Self::default(); + while !r.is_eof() { + match r.next_tag(bytes) { + Ok(10) => msg.nested = Some(r.read_message(bytes, mod_BazMessage::Nested::from_reader)?), + Ok(t) => { r.read_unknown(bytes, t)?; } + Err(e) => return Err(e), + } + } + Ok(msg) + } +} + +impl MessageWrite for BazMessage { + fn get_size(&self) -> usize { + self.nested.as_ref().map_or(0, |m| 1 + sizeof_var_length(m.get_size())) + } + + fn write_message(&self, r: &mut Writer) -> Result<()> { + if let Some(ref s) = self.nested { r.write_message_with_tag(10, s)?; } + Ok(()) + } +} + +pub mod mod_BazMessage { + +use super::*; + +#[derive(Debug, Default, PartialEq, Clone)] +pub struct Nested { + pub f_nested: i32, +} + +impl Nested { + pub fn from_reader(r: &mut BytesReader, bytes: &[u8]) -> Result { + let mut msg = Self::default(); + while !r.is_eof() { + match r.next_tag(bytes) { + Ok(8) => msg.f_nested = r.read_int32(bytes)?, + Ok(t) => { r.read_unknown(bytes, t)?; } + Err(e) => return Err(e), + } + } + Ok(msg) + } +} + +impl MessageWrite for Nested { + fn get_size(&self) -> usize { + 1 + sizeof_int32(self.f_nested) + } + + fn write_message(&self, r: &mut Writer) -> Result<()> { + r.write_int32_with_tag(8, self.f_nested)?; + Ok(()) + } +} + +} diff --git a/examples/codegen/data_types_import.rs b/examples/codegen/data_types_import.rs index ac93f5e6..db45eaef 100644 --- a/examples/codegen/data_types_import.rs +++ b/examples/codegen/data_types_import.rs @@ -4,8 +4,8 @@ #![allow(non_upper_case_globals)] #![allow(non_camel_case_types)] -pub mod a { -pub mod b { +pub mod mod_a { +pub mod mod_b { use std::io::{Write}; use quick_protobuf::{MessageWrite, BytesReader, Writer, Result}; diff --git a/examples/codegen_example.rs b/examples/codegen_example.rs index 763856ec..ca5d3462 100644 --- a/examples/codegen_example.rs +++ b/examples/codegen_example.rs @@ -5,7 +5,7 @@ mod codegen; use std::borrow::Cow; use codegen::data_types::FooMessage; -use codegen::data_types_import::a::b::ImportedMessage; +use codegen::data_types_import::mod_a::mod_b::ImportedMessage; use quick_protobuf::{BytesReader, Writer}; fn main() { From 8290c5750eee585c54e314744a97f3c767c763e0 Mon Sep 17 00:00:00 2001 From: Johann Tuffe Date: Mon, 6 Feb 2017 17:29:41 +0800 Subject: [PATCH 4/4] Set nested field in example --- examples/codegen/data_types.proto | 3 ++- examples/codegen/data_types.rs | 8 ++++++-- examples/codegen_example.rs | 19 ++++++++++++++++++- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/examples/codegen/data_types.proto b/examples/codegen/data_types.proto index c8467c71..bd4fb31d 100644 --- a/examples/codegen/data_types.proto +++ b/examples/codegen/data_types.proto @@ -31,7 +31,8 @@ message FooMessage { repeated int32 f_repeated_int32 = 19; repeated int32 f_repeated_packed_int32 = 20 [ packed = true ]; optional a.b.ImportedMessage f_imported = 21; - optional BazMessage.Nested f_nested = 22; + optional BazMessage f_baz = 22; + optional BazMessage.Nested f_nested = 23; } message BazMessage { diff --git a/examples/codegen/data_types.rs b/examples/codegen/data_types.rs index 398f7478..9a3c6474 100644 --- a/examples/codegen/data_types.rs +++ b/examples/codegen/data_types.rs @@ -86,6 +86,7 @@ pub struct FooMessage<'a> { pub f_repeated_int32: Vec, pub f_repeated_packed_int32: Vec, pub f_imported: Option, + pub f_baz: Option, pub f_nested: Option, } @@ -115,7 +116,8 @@ impl<'a> FooMessage<'a> { Ok(152) => msg.f_repeated_int32.push(r.read_int32(bytes)?), Ok(162) => msg.f_repeated_packed_int32 = r.read_packed(bytes, |r, bytes| r.read_int32(bytes))?, Ok(170) => msg.f_imported = Some(r.read_message(bytes, mod_a::mod_b::ImportedMessage::from_reader)?), - Ok(178) => msg.f_nested = Some(r.read_message(bytes, mod_BazMessage::Nested::from_reader)?), + Ok(178) => msg.f_baz = Some(r.read_message(bytes, BazMessage::from_reader)?), + Ok(186) => msg.f_nested = Some(r.read_message(bytes, mod_BazMessage::Nested::from_reader)?), Ok(t) => { r.read_unknown(bytes, t)?; } Err(e) => return Err(e), } @@ -147,6 +149,7 @@ impl<'a> MessageWrite for FooMessage<'a> { + self.f_repeated_int32.iter().map(|s| 2 + sizeof_int32(*s)).sum::() + if self.f_repeated_packed_int32.is_empty() { 0 } else { 2 + sizeof_var_length(self.f_repeated_packed_int32.iter().map(|s| sizeof_int32(*s)).sum::()) } + self.f_imported.as_ref().map_or(0, |m| 2 + sizeof_var_length(m.get_size())) + + self.f_baz.as_ref().map_or(0, |m| 2 + sizeof_var_length(m.get_size())) + self.f_nested.as_ref().map_or(0, |m| 2 + sizeof_var_length(m.get_size())) } @@ -172,7 +175,8 @@ impl<'a> MessageWrite for FooMessage<'a> { for s in &self.f_repeated_int32 { r.write_int32_with_tag(152, *s)? } r.write_packed_repeated_field_with_tag(162, &self.f_repeated_packed_int32, |r, m| r.write_int32(*m), &|m| sizeof_int32(*m))?; if let Some(ref s) = self.f_imported { r.write_message_with_tag(170, s)?; } - if let Some(ref s) = self.f_nested { r.write_message_with_tag(178, s)?; } + if let Some(ref s) = self.f_baz { r.write_message_with_tag(178, s)?; } + if let Some(ref s) = self.f_nested { r.write_message_with_tag(186, s)?; } Ok(()) } } diff --git a/examples/codegen_example.rs b/examples/codegen_example.rs index ca5d3462..f5fbb539 100644 --- a/examples/codegen_example.rs +++ b/examples/codegen_example.rs @@ -4,8 +4,12 @@ mod codegen; use std::borrow::Cow; -use codegen::data_types::FooMessage; +use codegen::data_types::{self, FooMessage}; + +// Imported fields contain package a.b, which is translated into +// mod_a::mod_b rust module use codegen::data_types_import::mod_a::mod_b::ImportedMessage; + use quick_protobuf::{BytesReader, Writer}; fn main() { @@ -14,10 +18,23 @@ fn main() { // // For the example we will leverage the `Default` derive of all messages let message = FooMessage { + + // Regular field work as expected, optional leverages on rust Option<> f_int32: Some(54), + + // strings are borrowed (Cow) f_string: Some(Cow::Borrowed("Hello world from example!")), + + // bytes too f_bytes: Some(Cow::Borrowed(b"I see you!")), + + // imported fields work as expected f_imported: Some(ImportedMessage { i: Some(true) }), + + // nested messages are encapsulated into a rust module mod_Message + f_nested: Some(data_types::mod_BazMessage::Nested { f_nested: 2 }), + + // Each message implements Default ... which makes it much easier ..FooMessage::default() };