From 7ba358945fbf551906f6fbe66f30d68e371136e9 Mon Sep 17 00:00:00 2001 From: Vitaly _Vi Shukela Date: Wed, 12 Jul 2023 19:54:01 +0200 Subject: [PATCH] 0.3.0: syn2 --- Cargo.lock | 37 +++-- Cargo.toml | 18 +-- crates/syn-file-expand-cli/Cargo.toml | 8 +- crates/syn-file-expand-cli/src/getcfgname.rs | 2 + crates/syn-file-expand-cli/src/loopify.rs | 6 +- crates/syn-file-expand-cli/src/undoc.rs | 8 +- src/attrs.rs | 158 +++++++------------ src/expand_impl.rs | 71 +++++---- tests/fullsource.rs | 4 +- 9 files changed, 142 insertions(+), 170 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e1a6d28..1b2cac5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -34,7 +34,7 @@ checksum = "729f9bd3449d77e7831a18abfb7ba2f99ee813dfd15b8c2167c9a54ba20aa99d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.95", ] [[package]] @@ -62,28 +62,28 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.1.10" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9e07e3a46d0771a8a06b5f4441527802830b43e679ba12f44960f48dd4c6803" +checksum = "9825a04601d60621feed79c4e6b56d65db77cdca55cef43b46b0de1096d1c282" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.18", ] [[package]] name = "proc-macro2" -version = "1.0.39" +version = "1.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c54b25569025b7fc9651de43004ae593a75ad88543b17178aa5e1b9c4f15f56f" +checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.18" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1" +checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" dependencies = [ "proc-macro2", ] @@ -124,28 +124,39 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn-file-expand" -version = "0.2.0" +version = "0.3.0" dependencies = [ "im-rc", "itertools", "prettyplease", "proc-macro2", "quote", - "syn", + "syn 2.0.18", "thiserror", ] [[package]] name = "syn-file-expand-cli" -version = "0.2.0" +version = "0.3.0" dependencies = [ "gumdrop", "prettyplease", "proc-macro2", "quote", - "syn", + "syn 2.0.18", "syn-file-expand", ] @@ -166,7 +177,7 @@ checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.95", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index c645bd4..04469c3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "syn-file-expand" -version = "0.2.0" +version = "0.3.0" edition = "2021" readme = "README.md" license = "MIT/Apache-2.0" @@ -14,21 +14,13 @@ include = ["src","Cargo.toml","tests", "examples", "resources", "README.md"] im-rc = "15.0.0" itertools = "0.10.3" proc-macro2 = "1.0.36" -quote = "1.0.15" # almost not needed, just for one tricky case -syn = { version = "1.0.88", features = ["full","extra-traits","printing"] } +quote = "1.0.15" +syn = { version = "2", features = ["full","extra-traits","printing"] } thiserror = "1.0.30" [dev-dependencies] -#pretty_assertions = "1.2.1" -prettyplease = "0.1.10" +#pretty_assertions = "1.4" +prettyplease = "0.2" [workspace] members = [".", "crates/*"] - -[profile.release] -opt-level = "s" -debug = 1 -lto = true -codegen-units = 1 -incremental = false -panic = 'abort' diff --git a/crates/syn-file-expand-cli/Cargo.toml b/crates/syn-file-expand-cli/Cargo.toml index 01e4188..19f63b0 100644 --- a/crates/syn-file-expand-cli/Cargo.toml +++ b/crates/syn-file-expand-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "syn-file-expand-cli" -version = "0.2.0" +version = "0.3.0" edition = "2021" readme = "README.md" @@ -14,12 +14,12 @@ include = ["src","Cargo.toml","README.md"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -syn-file-expand = {version = "0.2.0", path = "../.."} +syn-file-expand = {version = "0.3.0", path = "../.."} quote = "1.0.15" -syn = { version = "1.0.88", features = ["full"] } +syn = { version = "2", features = ["full"] } proc-macro2 = "1.0.36" gumdrop = "0.8.1" -prettyplease = { version = "0.1.10", optional = true } +prettyplease = { version = "0.2", optional = true } [features] default = ["prettyplease"] diff --git a/crates/syn-file-expand-cli/src/getcfgname.rs b/crates/syn-file-expand-cli/src/getcfgname.rs index c59f6ef..9f4dccc 100644 --- a/crates/syn-file-expand-cli/src/getcfgname.rs +++ b/crates/syn-file-expand-cli/src/getcfgname.rs @@ -28,6 +28,7 @@ pub(crate) fn get_env_name(input: proc_macro2::TokenStream) -> String { syn::Lit::Float(_) => "".to_owned(), syn::Lit::Bool(x) => format!("{}", x.value), syn::Lit::Verbatim(_) => "".to_owned(), + _ => "".to_owned(), } } else { eprintln!("Failed to parse a literal in a cfg `{}`", x); @@ -75,6 +76,7 @@ pub(crate) fn get_cli_name(input: proc_macro2::TokenStream) -> String { syn::Lit::Float(_) => "".to_owned(), syn::Lit::Bool(x) => format!("{}", x.value), syn::Lit::Verbatim(_) => "".to_owned(), + _ => "".to_owned(), } } else { eprintln!("Failed to parse a literal in a cfg `{}`", x); diff --git a/crates/syn-file-expand-cli/src/loopify.rs b/crates/syn-file-expand-cli/src/loopify.rs index 216a198..28a4725 100644 --- a/crates/syn-file-expand-cli/src/loopify.rs +++ b/crates/syn-file-expand-cli/src/loopify.rs @@ -12,7 +12,7 @@ fn loopify_block(block: &mut syn::Block) { brace_token: syn::token::Brace::default(), stmts: vec![], }, - }))); + }), None)); } fn loopify_expr(expr: &mut syn::Expr) { @@ -42,7 +42,7 @@ fn loopify_item(item: &mut syn::Item) { loopify_expr(&mut z.1); } } - syn::TraitItem::Method(y) => { + syn::TraitItem::Fn(y) => { if let Some(z) = &mut y.default { loopify_block(z); } @@ -69,7 +69,7 @@ fn loopify_item(item: &mut syn::Item) { syn::ImplItem::Const(y) => { loopify_expr(&mut y.expr); } - syn::ImplItem::Method(y) => { + syn::ImplItem::Fn(y) => { loopify_block(&mut y.block); } syn::ImplItem::Macro(y) => { diff --git a/crates/syn-file-expand-cli/src/undoc.rs b/crates/syn-file-expand-cli/src/undoc.rs index 8c245ec..904a934 100644 --- a/crates/syn-file-expand-cli/src/undoc.rs +++ b/crates/syn-file-expand-cli/src/undoc.rs @@ -1,8 +1,8 @@ fn undoc_attrs(attrs: &mut Vec) { attrs.retain(|attr| { - match attr.path.get_ident().map(|x|x.to_string()).as_deref() { - Some("doc") => false, + match &attr.meta { + syn::Meta::NameValue(x) => !x.path.is_ident("doc"), _ => true, } }); @@ -17,7 +17,7 @@ fn undoc_item(item: &mut syn::Item) { syn::TraitItem::Const(y) => { undoc_attrs(&mut y.attrs) } - syn::TraitItem::Method(y) => { + syn::TraitItem::Fn(y) => { undoc_attrs(&mut y.attrs) } syn::TraitItem::Type(y) => undoc_attrs(&mut y.attrs), @@ -42,7 +42,7 @@ fn undoc_item(item: &mut syn::Item) { for subitem in &mut x.items { match subitem { syn::ImplItem::Const(y) => undoc_attrs(&mut y.attrs), - syn::ImplItem::Method(y) =>undoc_attrs(&mut y.attrs), + syn::ImplItem::Fn(y) =>undoc_attrs(&mut y.attrs), syn::ImplItem::Macro(y) =>undoc_attrs(&mut y.attrs), syn::ImplItem::Type(y) =>undoc_attrs(&mut y.attrs), syn::ImplItem::Verbatim(_) => (), diff --git a/src/attrs.rs b/src/attrs.rs index f8064bc..7b41678 100644 --- a/src/attrs.rs +++ b/src/attrs.rs @@ -1,137 +1,89 @@ use proc_macro2; use proc_macro2::TokenStream; +use quote::ToTokens; +use syn::MacroDelimiter; +use syn::Meta; +use syn::MetaList; +use syn::MetaNameValue; +use syn::Token; +use syn::punctuated::Punctuated; -use crate::ErrorCase; use super::AttrParseError; -use super::Error; use std::path::PathBuf; -use proc_macro2::TokenTree; - -pub(crate) fn extract_path_from_attr( - tt: Vec, - mod_syn_path: &syn::Path, -) -> Result { - let ape = |x|Err(Error { - module: mod_syn_path.clone(), - inner: ErrorCase::AttrParseError(x), - }); - if tt.len() != 2 { - return Err(Error { - module: mod_syn_path.clone(), - inner: ErrorCase::AttrParseError(AttrParseError::NotExactlyTwoTokens), - }); - } - match &tt[0] { - TokenTree::Punct(x) if x.as_char() == '=' => (), - _ => { - return ape(AttrParseError::FirstTokenIsNotEqualSign); - } - } - match &tt[1] { - TokenTree::Literal(_) => (), - _ => { - return ape(AttrParseError::SecondTokenIsNotStringLiteral) - } - } - let ts = TokenStream::from(tt[1].clone()); - let tslit: syn::Lit = syn::parse2(ts).map_err(|e| Error { - module: mod_syn_path.clone(), - inner: ErrorCase::SynParseError(e), - })?; - let explicit_path = match tslit { - syn::Lit::Str(x) => PathBuf::from(x.value()), - _ => { - return ape(AttrParseError::SecondTokenIsNotStringLiteral) - } - }; - Ok(explicit_path) -} pub(crate) fn read_and_process_attributes( input_attrs: &[syn::Attribute], - path_attrs: &mut Vec<(Vec, Option)>, + path_attrs: &mut Vec<(PathBuf, Option)>, attrs: &mut Vec, cfg_attrs: &mut Vec, ) -> Result<(), AttrParseError> { for attr in input_attrs { - match &attr.path { - x if x.get_ident().map(|x| x.to_string()) == Some("cfg".to_owned()) => { - let tt = Vec::::from_iter(attr.tokens.clone()); - if tt.len() != 1 { + match &attr.meta { + Meta::List(MetaList { path, delimiter, tokens }) if path.is_ident("cfg") => { + if !matches!(delimiter, MacroDelimiter::Paren(..)) { return Err(AttrParseError::MalformedCfg); } - let g = match tt.into_iter().next().unwrap() { - TokenTree::Group(g) if g.delimiter() == proc_macro2::Delimiter::Parenthesis => g, - _ => return Err(AttrParseError::MalformedCfg), + cfg_attrs.push(tokens.clone()); + } + Meta::NameValue(MetaNameValue { path, eq_token: _, value }) if path.is_ident("path") => { + let path = match value { + syn::Expr::Lit(lit) => match &lit.lit { + syn::Lit::Str(x) => x.value(), + _ => return Err(AttrParseError::SecondTokenIsNotStringLiteral), + } + _ => return Err(AttrParseError::SecondTokenIsNotStringLiteral), }; - cfg_attrs.push(g.stream()); + path_attrs.push((PathBuf::from(path), None)); + } + Meta::Path(path) | Meta::List(MetaList { path, .. }) if path.is_ident("path") => { + return Err(AttrParseError::FirstTokenIsNotEqualSign); } - x if x.get_ident().map(|x| x.to_string()) == Some("path".to_owned()) => { - let tt = Vec::::from_iter(attr.tokens.clone()); - path_attrs.push((tt, None)); + + Meta::Path(path) | Meta::NameValue(MetaNameValue { path, .. }) if path.is_ident("cfg_attr") => { + return Err(AttrParseError::CfgAttrNotRoundGroup); } - x if x.get_ident().map(|x| x.to_string()) == Some("cfg_attr".to_owned()) => { - let tt = Vec::::from_iter(attr.tokens.clone()); - if tt.len() != 1 { + + Meta::List(MetaList { path, delimiter, .. }) if path.is_ident("cfg_attr") => { + if !matches!(delimiter, MacroDelimiter::Paren(..)) { return Err(AttrParseError::CfgAttrNotRoundGroup); } - let t = tt.into_iter().next().unwrap(); - let g = match t { - TokenTree::Group(g) - if g.delimiter() == proc_macro2::Delimiter::Parenthesis => - { - g - } - _ => { - return Err(AttrParseError::CfgAttrNotRoundGroup); - } + + let Ok(nested) = attr.parse_args_with(Punctuated::::parse_terminated) else { + return Err(AttrParseError::MalformedCfg) }; - let inner = Vec::::from_iter(g.stream()); - if inner.len() < 3 { - return Err(AttrParseError::CfgAttrNotTwoParams); - } - let mut ts_before_comma = TokenStream::new(); - let mut ts_after_comma = Vec::::new(); - let mut comma_encountered = false; - for t in inner.into_iter() { - match t { - TokenTree::Punct(p) if p.as_char() == ',' => { - if comma_encountered { - return Err(AttrParseError::CfgAttrNotTwoParams); - } - comma_encountered = true; - } - x => { - if comma_encountered { - ts_after_comma.push(x) - } else { - ts_before_comma.extend(std::iter::once(x)) - } - } - } - } - if !comma_encountered { + + if nested.len() != 2 { return Err(AttrParseError::CfgAttrNotTwoParams); } - let mut pathy = false; - if !ts_after_comma.is_empty() { - match &ts_after_comma[0] { - TokenTree::Ident(i) if i.to_string() == "path" => pathy = true, - _ => (), - } - } + let condition = &nested[0]; + let potential_path = &nested[1]; - if pathy { - path_attrs.push((ts_after_comma[1..].to_vec(), Some(ts_before_comma))); - } else { + if ! potential_path.path().is_ident("path") { attrs.push(attr.clone()); + continue; } + + let path = match potential_path { + Meta::NameValue(MetaNameValue { path: _, eq_token: _, value }) => { + match value { + syn::Expr::Lit(lit) => match &lit.lit { + syn::Lit::Str(x) => x.value(), + _ => return Err(AttrParseError::SecondTokenIsNotStringLiteral), + } + _ => return Err(AttrParseError::SecondTokenIsNotStringLiteral), + } + } + _ => return Err(AttrParseError::FirstTokenIsNotEqualSign), + }; + + path_attrs.push((PathBuf::from(path), Some(condition.to_token_stream()))); } + _ => attrs.push(attr.clone()), } } diff --git a/src/expand_impl.rs b/src/expand_impl.rs index c780c32..ffdd2c2 100644 --- a/src/expand_impl.rs +++ b/src/expand_impl.rs @@ -1,8 +1,9 @@ use std::path::PathBuf; use im_rc::Vector; -use proc_macro2::{TokenStream, TokenTree}; -use syn::{punctuated::Punctuated, spanned::Spanned, MetaList}; +use proc_macro2::TokenStream; +use quote::ToTokens; +use syn::{punctuated::Punctuated, spanned::Spanned, MetaList, Token}; use crate::{attrs, Error, ErrorCase, Resolver}; @@ -65,6 +66,7 @@ pub(crate) fn expand_impl( module_file_mod.push("mod.rs"); module_file_nomod.push(&chunk_rs); + // Used mostly for error reporting let mod_syn_path = syn::Path { leading_colon: None, segments: syn::punctuated::Punctuated::from_iter(inner_stack.iter().map(|x| { @@ -88,14 +90,15 @@ pub(crate) fn expand_impl( dirs_attr: Vector, /// Root path to use for recursive expansions for inner modules without `#[path]` ("natural" path) dirs_nat: Vector, - cfg: Option, + /// Special additional `#[cfg]` attribute that is synthesized and injected into expanded module in multi-module mode + injected_cfg: Option, } // outer level of Vec: expansion results based on multiple path attributes or `name/mod.rs` vs `name.rs` distinction. // inner level of Option: whether the expansion resulted in actual code or failure to open a file let mut expansion_candidates: Vec = Vec::with_capacity(1); - let mut path_attrs: Vec<(Vec, Option)> = Vec::new(); + let mut path_attrs: Vec<(PathBuf, Option)> = Vec::new(); let mut cfg_attrs: Vec = Vec::new(); attrs::read_and_process_attributes( &item_mod.attrs, @@ -121,13 +124,11 @@ pub(crate) fn expand_impl( let mut need_to_try_natural_file_locations = true; let mut accumulated_cfgs = Vec::::new(); - for (tt, cfg) in path_attrs { - if cfg.is_none() && !expansion_candidates.is_empty() && !multimodule_mode { + for (explicit_path, condition_in_cfg_attr_path) in path_attrs { + if condition_in_cfg_attr_path.is_none() && !expansion_candidates.is_empty() && !multimodule_mode { return Err(err(ErrorCase::MultipleExplicitPathsSpecifiedForOneModule)); } - let explicit_path = attrs::extract_path_from_attr(tt, &mod_syn_path)?; - let mut module_file_explicit = PathBuf::with_capacity(len_hint); for x in &dirs_attr { @@ -140,7 +141,7 @@ pub(crate) fn expand_impl( dirs_candidate.push_back(parent.to_owned()); } - if let Some(cfg) = cfg { + if let Some(cfg) = condition_in_cfg_attr_path { let cfg: syn::Meta = syn::parse2(cfg).map_err(|e| err(ErrorCase::SynParseError(e)))?; if resolver @@ -158,7 +159,7 @@ pub(crate) fn expand_impl( result, dirs_attr: dirs_candidate.clone(), dirs_nat: dirs_candidate, - cfg: Some(cfg.clone()), + injected_cfg: Some(cfg.clone()), }); accumulated_cfgs.push(cfg); } @@ -169,7 +170,7 @@ pub(crate) fn expand_impl( result, dirs_attr: dirs_candidate.clone(), dirs_nat: dirs_candidate, - cfg: None, + injected_cfg: None, }); } } @@ -226,21 +227,26 @@ pub(crate) fn expand_impl( (_, Err(e)) => return Err(e), }; let some_span = item_mod.span(); + let throwaway_group = proc_macro2::Group::new(proc_macro2::Delimiter::Bracket, Default::default()); + // I'm not sure what to do with spans of newly generated tokens. + let some_delim_span = throwaway_group.delim_span(); let cfg = if multimodule_mode && !accumulated_cfgs.is_empty() { + let tokens_inside_any : TokenStream = Punctuated::<_,Token![,]>::from_iter( + accumulated_cfgs + .iter() + .map(|x| x.clone()), + ).into_token_stream(); + let tokens_inside_not : TokenStream = syn::Meta::List( + MetaList { + path: simple_path(some_span, "any"), + delimiter: syn::MacroDelimiter::Paren(syn::token::Paren { span: some_delim_span }), + tokens: tokens_inside_any, + }, + ).into_token_stream(); Some(syn::Meta::List(MetaList { path: simple_path(some_span, "not"), - paren_token: syn::token::Paren { span: some_span }, - nested: Punctuated::from_iter([syn::NestedMeta::Meta(syn::Meta::List( - MetaList { - path: simple_path(some_span, "any"), - paren_token: syn::token::Paren { span: some_span }, - nested: Punctuated::from_iter( - accumulated_cfgs - .iter() - .map(|x| syn::NestedMeta::Meta(x.clone())), - ), - }, - ))]), + delimiter: syn::MacroDelimiter::Paren(syn::token::Paren { span: some_delim_span }), + tokens: tokens_inside_not, })) } else { None @@ -249,7 +255,7 @@ pub(crate) fn expand_impl( result, dirs_attr, dirs_nat, - cfg, + injected_cfg: cfg, }); } @@ -262,7 +268,7 @@ pub(crate) fn expand_impl( result, dirs_attr, dirs_nat, - cfg, + injected_cfg: cfg, } in expansion_candidates.into_iter() { if let Some(inner) = result { @@ -271,6 +277,8 @@ pub(crate) fn expand_impl( let mut attrs_copy = attrs.clone(); let some_span = item_mod.span(); + let throwaway_group = proc_macro2::Group::new(proc_macro2::Delimiter::Bracket, Default::default()); + let some_delim_span = throwaway_group.delim_span(); if let Some(cfg) = cfg { if multimodule_mode { attrs_copy.push(syn::Attribute { @@ -278,9 +286,12 @@ pub(crate) fn expand_impl( // just plugging whatever matches the signature and what I have found the first. pound_token: syn::token::Pound { spans: [some_span] }, style: syn::AttrStyle::Outer, - bracket_token: syn::token::Bracket { span: some_span }, - path: simple_path(some_span, "cfg"), - tokens: quote::quote!((#cfg)), + bracket_token: syn::token::Bracket { span: some_delim_span }, + meta : syn::Meta::List(syn::MetaList { + path: simple_path(some_span, "cfg"), + delimiter: syn::MacroDelimiter::Paren(syn::token::Paren { span: some_delim_span }), + tokens: cfg.into_token_stream(), + }), }) } } @@ -290,6 +301,9 @@ pub(crate) fn expand_impl( } //dbg!(&inner_stack, &dirs_nat, &dirs_attr); + + // recursive call to process nested modules + expand_impl( &mut inner_items, resolver, @@ -311,6 +325,7 @@ pub(crate) fn expand_impl( inner_items, )), semi: None, + unsafety: item_mod.unsafety, }; if !multimodule_mode { diff --git a/tests/fullsource.rs b/tests/fullsource.rs index 4b8277a..cd13baa 100644 --- a/tests/fullsource.rs +++ b/tests/fullsource.rs @@ -1,5 +1,5 @@ -use quote::{quote as q}; -//use pretty_assertions::{assert_eq}; +use quote::quote as q; +//use pretty_assertions::assert_eq; #[test] fn fullsource_plain() {