From 55082ce41334d9685d619182e71b84b495a6aff8 Mon Sep 17 00:00:00 2001 From: Aaron Hill Date: Fri, 21 Aug 2020 18:51:23 -0400 Subject: [PATCH] Attach `TokenStream` to `ast::Path` --- compiler/rustc_ast/src/ast.rs | 5 +++-- compiler/rustc_ast/src/attr/mod.rs | 2 +- compiler/rustc_ast/src/mut_visit.rs | 2 +- compiler/rustc_ast_lowering/src/item.rs | 13 ++++++++----- compiler/rustc_expand/src/build.rs | 2 +- compiler/rustc_expand/src/placeholders.rs | 2 +- compiler/rustc_parse/src/lib.rs | 1 + compiler/rustc_parse/src/parser/diagnostics.rs | 2 +- compiler/rustc_parse/src/parser/item.rs | 2 +- compiler/rustc_parse/src/parser/nonterminal.rs | 10 +++++++++- compiler/rustc_parse/src/parser/path.rs | 9 ++++++--- compiler/rustc_resolve/src/diagnostics.rs | 2 +- compiler/rustc_resolve/src/late.rs | 2 +- compiler/rustc_resolve/src/late/diagnostics.rs | 6 ++++-- compiler/rustc_resolve/src/lib.rs | 2 ++ 15 files changed, 41 insertions(+), 21 deletions(-) diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs index 6db84f0dbd398..80c8cdf703cf3 100644 --- a/compiler/rustc_ast/src/ast.rs +++ b/compiler/rustc_ast/src/ast.rs @@ -96,6 +96,7 @@ pub struct Path { /// The segments in the path: the things separated by `::`. /// Global paths begin with `kw::PathRoot`. pub segments: Vec, + pub tokens: Option, } impl PartialEq for Path { @@ -117,7 +118,7 @@ impl Path { // Convert a span and an identifier to the corresponding // one-segment path. pub fn from_ident(ident: Ident) -> Path { - Path { segments: vec![PathSegment::from_ident(ident)], span: ident.span } + Path { segments: vec![PathSegment::from_ident(ident)], span: ident.span, tokens: None } } pub fn is_global(&self) -> bool { @@ -1069,7 +1070,7 @@ pub struct Expr { // `Expr` is used a lot. Make sure it doesn't unintentionally get bigger. #[cfg(target_arch = "x86_64")] -rustc_data_structures::static_assert_size!(Expr, 104); +rustc_data_structures::static_assert_size!(Expr, 112); impl Expr { /// Returns `true` if this expression would be valid somewhere that expects a value; diff --git a/compiler/rustc_ast/src/attr/mod.rs b/compiler/rustc_ast/src/attr/mod.rs index 33bd93352316a..2782869fb885a 100644 --- a/compiler/rustc_ast/src/attr/mod.rs +++ b/compiler/rustc_ast/src/attr/mod.rs @@ -415,7 +415,7 @@ impl MetaItem { } } let span = span.with_hi(segments.last().unwrap().ident.span.hi()); - Path { span, segments } + Path { span, segments, tokens: None } } Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. })) => match *nt { token::Nonterminal::NtMeta(ref item) => return item.meta(item.path.span), diff --git a/compiler/rustc_ast/src/mut_visit.rs b/compiler/rustc_ast/src/mut_visit.rs index 3ba8ff65ee514..f6ac134bb1671 100644 --- a/compiler/rustc_ast/src/mut_visit.rs +++ b/compiler/rustc_ast/src/mut_visit.rs @@ -513,7 +513,7 @@ pub fn noop_visit_ident(Ident { name: _, span }: &mut Ident, vis: vis.visit_span(span); } -pub fn noop_visit_path(Path { segments, span }: &mut Path, vis: &mut T) { +pub fn noop_visit_path(Path { segments, span, tokens: _ }: &mut Path, vis: &mut T) { vis.visit_span(span); for PathSegment { ident, id, args } in segments { vis.visit_ident(ident); diff --git a/compiler/rustc_ast_lowering/src/item.rs b/compiler/rustc_ast_lowering/src/item.rs index f3309afec7d6b..6a3b9c22597ae 100644 --- a/compiler/rustc_ast_lowering/src/item.rs +++ b/compiler/rustc_ast_lowering/src/item.rs @@ -251,7 +251,7 @@ impl<'hir> LoweringContext<'_, 'hir> { ItemKind::ExternCrate(orig_name) => hir::ItemKind::ExternCrate(orig_name), ItemKind::Use(ref use_tree) => { // Start with an empty prefix. - let prefix = Path { segments: vec![], span: use_tree.span }; + let prefix = Path { segments: vec![], span: use_tree.span, tokens: None }; self.lower_use_tree(use_tree, &prefix, id, vis, ident, attrs) } @@ -488,7 +488,7 @@ impl<'hir> LoweringContext<'_, 'hir> { *ident = tree.ident(); // First, apply the prefix to the path. - let mut path = Path { segments, span: path.span }; + let mut path = Path { segments, span: path.span, tokens: None }; // Correctly resolve `self` imports. if path.segments.len() > 1 @@ -540,8 +540,11 @@ impl<'hir> LoweringContext<'_, 'hir> { hir::ItemKind::Use(path, hir::UseKind::Single) } UseTreeKind::Glob => { - let path = - self.lower_path(id, &Path { segments, span: path.span }, ParamMode::Explicit); + let path = self.lower_path( + id, + &Path { segments, span: path.span, tokens: None }, + ParamMode::Explicit, + ); hir::ItemKind::Use(path, hir::UseKind::Glob) } UseTreeKind::Nested(ref trees) => { @@ -569,7 +572,7 @@ impl<'hir> LoweringContext<'_, 'hir> { // for that we return the `{}` import (called the // `ListStem`). - let prefix = Path { segments, span: prefix.span.to(path.span) }; + let prefix = Path { segments, span: prefix.span.to(path.span), tokens: None }; // Add all the nested `PathListItem`s to the HIR. for &(ref use_tree, id) in trees { diff --git a/compiler/rustc_expand/src/build.rs b/compiler/rustc_expand/src/build.rs index c57719d67163c..0fcffe77d45d3 100644 --- a/compiler/rustc_expand/src/build.rs +++ b/compiler/rustc_expand/src/build.rs @@ -46,7 +46,7 @@ impl<'a> ExtCtxt<'a> { id: ast::DUMMY_NODE_ID, args, }); - ast::Path { span, segments } + ast::Path { span, segments, tokens: None } } pub fn ty_mt(&self, ty: P, mutbl: ast::Mutability) -> ast::MutTy { diff --git a/compiler/rustc_expand/src/placeholders.rs b/compiler/rustc_expand/src/placeholders.rs index f642558d5e07a..530e67a134a35 100644 --- a/compiler/rustc_expand/src/placeholders.rs +++ b/compiler/rustc_expand/src/placeholders.rs @@ -18,7 +18,7 @@ pub fn placeholder( ) -> AstFragment { fn mac_placeholder() -> ast::MacCall { ast::MacCall { - path: ast::Path { span: DUMMY_SP, segments: Vec::new() }, + path: ast::Path { span: DUMMY_SP, segments: Vec::new(), tokens: None }, args: P(ast::MacArgs::Empty), prior_type_ascription: None, } diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs index bf954783d8273..69f63feac78ee 100644 --- a/compiler/rustc_parse/src/lib.rs +++ b/compiler/rustc_parse/src/lib.rs @@ -278,6 +278,7 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke Some(tokenstream::TokenTree::token(token::Lifetime(ident.name), ident.span).into()) } Nonterminal::NtMeta(ref attr) => attr.tokens.clone(), + Nonterminal::NtPath(ref path) => path.tokens.clone(), Nonterminal::NtTT(ref tt) => Some(tt.clone().into()), Nonterminal::NtExpr(ref expr) | Nonterminal::NtLiteral(ref expr) => { if expr.tokens.is_none() { diff --git a/compiler/rustc_parse/src/parser/diagnostics.rs b/compiler/rustc_parse/src/parser/diagnostics.rs index 364c859147ac5..e2a735188f95c 100644 --- a/compiler/rustc_parse/src/parser/diagnostics.rs +++ b/compiler/rustc_parse/src/parser/diagnostics.rs @@ -901,7 +901,7 @@ impl<'a> Parser<'a> { ) -> PResult<'a, P> { self.expect(&token::ModSep)?; - let mut path = ast::Path { segments: Vec::new(), span: DUMMY_SP }; + let mut path = ast::Path { segments: Vec::new(), span: DUMMY_SP, tokens: None }; self.parse_path_segments(&mut path.segments, T::PATH_STYLE)?; path.span = ty_span.to(self.prev_token.span); diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs index 1c4bb4532ebb4..6d90d1128227a 100644 --- a/compiler/rustc_parse/src/parser/item.rs +++ b/compiler/rustc_parse/src/parser/item.rs @@ -787,7 +787,7 @@ impl<'a> Parser<'a> { fn parse_use_tree(&mut self) -> PResult<'a, UseTree> { let lo = self.token.span; - let mut prefix = ast::Path { segments: Vec::new(), span: lo.shrink_to_lo() }; + let mut prefix = ast::Path { segments: Vec::new(), span: lo.shrink_to_lo(), tokens: None }; let kind = if self.check(&token::OpenDelim(token::Brace)) || self.check(&token::BinOp(token::Star)) || self.is_import_coupler() diff --git a/compiler/rustc_parse/src/parser/nonterminal.rs b/compiler/rustc_parse/src/parser/nonterminal.rs index 8717f4b92451e..84e887e74a7a4 100644 --- a/compiler/rustc_parse/src/parser/nonterminal.rs +++ b/compiler/rustc_parse/src/parser/nonterminal.rs @@ -168,7 +168,15 @@ impl<'a> Parser<'a> { return Err(self.struct_span_err(self.token.span, msg)); } } - NonterminalKind::Path => token::NtPath(self.parse_path(PathStyle::Type)?), + NonterminalKind::Path => { + let (mut path, tokens) = + self.collect_tokens(|this| this.parse_path(PathStyle::Type))?; + // We have have eaten an NtPath, which could already have tokens + if path.tokens.is_none() { + path.tokens = Some(tokens); + } + token::NtPath(path) + } NonterminalKind::Meta => { let (mut attr, tokens) = self.collect_tokens(|this| this.parse_attr_item())?; // We may have eaten a nonterminal, which could already have tokens diff --git a/compiler/rustc_parse/src/parser/path.rs b/compiler/rustc_parse/src/parser/path.rs index 54b4df8613f70..66ce015d02e4c 100644 --- a/compiler/rustc_parse/src/parser/path.rs +++ b/compiler/rustc_parse/src/parser/path.rs @@ -64,7 +64,7 @@ impl<'a> Parser<'a> { path_span = path_lo.to(self.prev_token.span); } else { path_span = self.token.span.to(self.token.span); - path = ast::Path { segments: Vec::new(), span: path_span }; + path = ast::Path { segments: Vec::new(), span: path_span, tokens: None }; } // See doc comment for `unmatched_angle_bracket_count`. @@ -81,7 +81,10 @@ impl<'a> Parser<'a> { let qself = QSelf { ty, path_span, position: path.segments.len() }; self.parse_path_segments(&mut path.segments, style)?; - Ok((qself, Path { segments: path.segments, span: lo.to(self.prev_token.span) })) + Ok(( + qself, + Path { segments: path.segments, span: lo.to(self.prev_token.span), tokens: None }, + )) } /// Recover from an invalid single colon, when the user likely meant a qualified path. @@ -144,7 +147,7 @@ impl<'a> Parser<'a> { } self.parse_path_segments(&mut segments, style)?; - Ok(Path { segments, span: lo.to(self.prev_token.span) }) + Ok(Path { segments, span: lo.to(self.prev_token.span), tokens: None }) } pub(super) fn parse_path_segments( diff --git a/compiler/rustc_resolve/src/diagnostics.rs b/compiler/rustc_resolve/src/diagnostics.rs index 48e1068b8daad..7a0503d68f348 100644 --- a/compiler/rustc_resolve/src/diagnostics.rs +++ b/compiler/rustc_resolve/src/diagnostics.rs @@ -794,7 +794,7 @@ impl<'a> Resolver<'a> { } segms.push(ast::PathSegment::from_ident(ident)); - let path = Path { span: name_binding.span, segments: segms }; + let path = Path { span: name_binding.span, segments: segms, tokens: None }; let did = match res { Res::Def(DefKind::Ctor(..), did) => this.parent(did), _ => res.opt_def_id(), diff --git a/compiler/rustc_resolve/src/late.rs b/compiler/rustc_resolve/src/late.rs index 2b2123e295d09..dda084214b8f4 100644 --- a/compiler/rustc_resolve/src/late.rs +++ b/compiler/rustc_resolve/src/late.rs @@ -1967,7 +1967,7 @@ impl<'a, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> { if qself.is_none() { let path_seg = |seg: &Segment| PathSegment::from_ident(seg.ident); - let path = Path { segments: path.iter().map(path_seg).collect(), span }; + let path = Path { segments: path.iter().map(path_seg).collect(), span, tokens: None }; if let Ok((_, res)) = self.r.resolve_macro_path(&path, None, &self.parent_scope, false, false) { diff --git a/compiler/rustc_resolve/src/late/diagnostics.rs b/compiler/rustc_resolve/src/late/diagnostics.rs index a10754f2c0c78..9f631705b2bf8 100644 --- a/compiler/rustc_resolve/src/late/diagnostics.rs +++ b/compiler/rustc_resolve/src/late/diagnostics.rs @@ -83,6 +83,7 @@ fn import_candidate_to_enum_paths(suggestion: &ImportSuggestion) -> (String, Str let enum_path = ast::Path { span: suggestion.path.span, segments: suggestion.path.segments[0..path_len - 1].to_vec(), + tokens: None, }; let enum_path_string = path_names_to_string(&enum_path); @@ -1065,7 +1066,8 @@ impl<'a> LateResolutionVisitor<'a, '_, '_> { path_segments.push(ast::PathSegment::from_ident(ident)); let module_def_id = module.def_id().unwrap(); if module_def_id == def_id { - let path = Path { span: name_binding.span, segments: path_segments }; + let path = + Path { span: name_binding.span, segments: path_segments, tokens: None }; result = Some(( module, ImportSuggestion { @@ -1095,7 +1097,7 @@ impl<'a> LateResolutionVisitor<'a, '_, '_> { if let Res::Def(DefKind::Variant, _) = name_binding.res() { let mut segms = enum_import_suggestion.path.segments.clone(); segms.push(ast::PathSegment::from_ident(ident)); - variants.push(Path { span: name_binding.span, segments: segms }); + variants.push(Path { span: name_binding.span, segments: segms, tokens: None }); } }); variants diff --git a/compiler/rustc_resolve/src/lib.rs b/compiler/rustc_resolve/src/lib.rs index 848f1c116ebcb..c7913e8445531 100644 --- a/compiler/rustc_resolve/src/lib.rs +++ b/compiler/rustc_resolve/src/lib.rs @@ -3189,6 +3189,7 @@ impl<'a> Resolver<'a> { .chain(path_str.split("::").skip(1).map(Ident::from_str)) .map(|i| self.new_ast_path_segment(i)) .collect(), + tokens: None, } } else { ast::Path { @@ -3198,6 +3199,7 @@ impl<'a> Resolver<'a> { .map(Ident::from_str) .map(|i| self.new_ast_path_segment(i)) .collect(), + tokens: None, } }; let module = self.get_module(module_id);