From 1fe2c0324006165b0c39ece0ccd7509e19583054 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Fri, 30 Nov 2018 10:02:04 +1100 Subject: [PATCH] Remove `tokenstream::Delimited`. Because it's an extra type layer that doesn't really help; in a couple of places it actively gets in the way, and overall removing it makes the code nicer. It does, however, move `tokenstream::TokenTree` further away from the `TokenTree` in `quote.rs`. More importantly, this change reduces the size of `TokenStream` from 48 bytes to 40 bytes on x86-64, which is enough to slightly reduce instruction counts on numerous benchmarks, the best by 1.5%. Note that `open_tt` and `close_tt` have gone from being methods on `Delimited` to associated methods of `TokenTree`. --- src/librustc/hir/lowering.rs | 10 +-- src/librustc/ich/impls_syntax.rs | 6 +- src/librustc_lint/builtin.rs | 4 +- src/libsyntax/ast.rs | 2 +- src/libsyntax/attr/mod.rs | 15 ++-- src/libsyntax/ext/expand.rs | 4 +- src/libsyntax/ext/quote.rs | 32 ++++---- src/libsyntax/ext/tt/quoted.rs | 16 ++-- src/libsyntax/ext/tt/transcribe.rs | 11 +-- src/libsyntax/fold.rs | 8 +- src/libsyntax/parse/lexer/tokentrees.rs | 9 ++- src/libsyntax/parse/mod.rs | 61 ++++++++------- src/libsyntax/parse/parser.rs | 80 ++++++++++---------- src/libsyntax/parse/token.rs | 7 +- src/libsyntax/print/pprust.rs | 8 +- src/libsyntax/tokenstream.rs | 98 +++++++++++-------------- src/libsyntax/visit.rs | 2 +- src/libsyntax_ext/proc_macro_server.rs | 12 ++- 18 files changed, 182 insertions(+), 203 deletions(-) diff --git a/src/librustc/hir/lowering.rs b/src/librustc/hir/lowering.rs index 6fd0ccb49b6d7..6958801d865bf 100644 --- a/src/librustc/hir/lowering.rs +++ b/src/librustc/hir/lowering.rs @@ -72,7 +72,7 @@ use syntax::ptr::P; use syntax::source_map::{self, respan, CompilerDesugaringKind, Spanned}; use syntax::std_inject; use syntax::symbol::{keywords, Symbol}; -use syntax::tokenstream::{Delimited, TokenStream, TokenTree}; +use syntax::tokenstream::{TokenStream, TokenTree}; use syntax::parse::token::Token; use syntax::visit::{self, Visitor}; use syntax_pos::{Span, MultiSpan}; @@ -1088,12 +1088,10 @@ impl<'a> LoweringContext<'a> { fn lower_token_tree(&mut self, tree: TokenTree) -> TokenStream { match tree { TokenTree::Token(span, token) => self.lower_token(token, span), - TokenTree::Delimited(span, delimited) => TokenTree::Delimited( + TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited( span, - Delimited { - delim: delimited.delim, - tts: self.lower_token_stream(delimited.tts.into()).into(), - }, + delim, + self.lower_token_stream(tts.into()).into(), ).into(), } } diff --git a/src/librustc/ich/impls_syntax.rs b/src/librustc/ich/impls_syntax.rs index db643e4fd726b..3211937d3ddc7 100644 --- a/src/librustc/ich/impls_syntax.rs +++ b/src/librustc/ich/impls_syntax.rs @@ -265,10 +265,10 @@ for tokenstream::TokenTree { span.hash_stable(hcx, hasher); hash_token(token, hcx, hasher); } - tokenstream::TokenTree::Delimited(span, ref delimited) => { + tokenstream::TokenTree::Delimited(span, delim, ref tts) => { span.hash_stable(hcx, hasher); - std_hash::Hash::hash(&delimited.delim, hasher); - for sub_tt in delimited.stream().trees() { + std_hash::Hash::hash(&delim, hasher); + for sub_tt in tts.stream().trees() { sub_tt.hash_stable(hcx, hasher); } } diff --git a/src/librustc_lint/builtin.rs b/src/librustc_lint/builtin.rs index e0e790e9f2302..7dd5d3c1cbc4c 100644 --- a/src/librustc_lint/builtin.rs +++ b/src/librustc_lint/builtin.rs @@ -1540,8 +1540,8 @@ impl KeywordIdents { } _ => {}, } - TokenTree::Delimited(_, ref delim) => { - self.check_tokens(cx, delim.tts.clone().into()) + TokenTree::Delimited(_, _, tts) => { + self.check_tokens(cx, tts.stream()) }, } } diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs index f25f456e3ae40..0792b2dc49c27 100644 --- a/src/libsyntax/ast.rs +++ b/src/libsyntax/ast.rs @@ -1235,7 +1235,7 @@ pub enum MacDelimiter { impl Mac_ { pub fn stream(&self) -> TokenStream { - self.tts.clone().into() + self.tts.stream() } } diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs index 7e8384bec6807..7723c15a266f1 100644 --- a/src/libsyntax/attr/mod.rs +++ b/src/libsyntax/attr/mod.rs @@ -34,7 +34,7 @@ use parse::token::{self, Token}; use ptr::P; use symbol::Symbol; use ThinVec; -use tokenstream::{TokenStream, TokenTree, Delimited, DelimSpan}; +use tokenstream::{TokenStream, TokenTree, DelimSpan}; use GLOBALS; use std::iter; @@ -549,10 +549,11 @@ impl MetaItemKind { } tokens.push(item.node.tokens()); } - TokenTree::Delimited(DelimSpan::from_single(span), Delimited { - delim: token::Paren, - tts: TokenStream::concat(tokens).into(), - }).into() + TokenTree::Delimited( + DelimSpan::from_single(span), + token::Paren, + TokenStream::concat(tokens).into(), + ).into() } } } @@ -570,9 +571,9 @@ impl MetaItemKind { None }; } - Some(TokenTree::Delimited(_, ref delimited)) if delimited.delim == token::Paren => { + Some(TokenTree::Delimited(_, delim, ref tts)) if delim == token::Paren => { tokens.next(); - delimited.stream() + tts.stream() } _ => return Some(MetaItemKind::Word), }; diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index 06e7b4657849f..f4e9a7e409359 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -622,9 +622,9 @@ impl<'a, 'b> MacroExpander<'a, 'b> { fn extract_proc_macro_attr_input(&self, tokens: TokenStream, span: Span) -> TokenStream { let mut trees = tokens.trees(); match trees.next() { - Some(TokenTree::Delimited(_, delim)) => { + Some(TokenTree::Delimited(_, _, tts)) => { if trees.next().is_none() { - return delim.tts.into() + return tts.into() } } Some(TokenTree::Token(..)) => {} diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs index 91818992fe1f1..c3497a17806b1 100644 --- a/src/libsyntax/ext/quote.rs +++ b/src/libsyntax/ext/quote.rs @@ -36,7 +36,7 @@ pub mod rt { use symbol::Symbol; use ThinVec; - use tokenstream::{self, DelimSpan, TokenTree, TokenStream}; + use tokenstream::{DelimSpan, TokenTree, TokenStream}; pub use parse::new_parser_from_tts; pub use syntax_pos::{BytePos, Span, DUMMY_SP, FileName}; @@ -246,9 +246,9 @@ pub mod rt { inner.push(self.tokens.clone()); let delim_span = DelimSpan::from_single(self.span); - r.push(TokenTree::Delimited(delim_span, tokenstream::Delimited { - delim: token::Bracket, tts: TokenStream::concat(inner).into() - })); + r.push(TokenTree::Delimited( + delim_span, token::Bracket, TokenStream::concat(inner).into() + )); r } } @@ -262,10 +262,9 @@ pub mod rt { impl ToTokens for () { fn to_tokens(&self, _cx: &ExtCtxt) -> Vec { - vec![TokenTree::Delimited(DelimSpan::dummy(), tokenstream::Delimited { - delim: token::Paren, - tts: TokenStream::empty().into(), - })] + vec![ + TokenTree::Delimited(DelimSpan::dummy(), token::Paren, TokenStream::empty().into()) + ] } } @@ -382,8 +381,6 @@ pub mod rt { // Replaces `Token::OpenDelim .. Token::CloseDelim` with `TokenTree::Delimited(..)`. pub fn unflatten(tts: Vec) -> Vec { - use tokenstream::Delimited; - let mut results = Vec::new(); let mut result = Vec::new(); let mut open_span = DUMMY_SP; @@ -395,10 +392,11 @@ pub fn unflatten(tts: Vec) -> Vec { } TokenTree::Token(span, token::CloseDelim(delim)) => { let delim_span = DelimSpan::from_pair(open_span, span); - let tree = TokenTree::Delimited(delim_span, Delimited { + let tree = TokenTree::Delimited( + delim_span, delim, - tts: result.into_iter().map(TokenStream::from).collect::().into(), - }); + result.into_iter().map(TokenStream::from).collect::().into(), + ); result = results.pop().unwrap(); result.push(tree); } @@ -758,10 +756,10 @@ fn statements_mk_tt(cx: &ExtCtxt, tt: &TokenTree, quoted: bool) -> Vec { - let mut stmts = statements_mk_tt(cx, &delimed.open_tt(span.open), false); - stmts.extend(statements_mk_tts(cx, delimed.stream())); - stmts.extend(statements_mk_tt(cx, &delimed.close_tt(span.close), false)); + TokenTree::Delimited(span, delim, ref tts) => { + let mut stmts = statements_mk_tt(cx, &TokenTree::open_tt(span.open, delim), false); + stmts.extend(statements_mk_tts(cx, tts.stream())); + stmts.extend(statements_mk_tt(cx, &TokenTree::close_tt(span.close, delim), false)); stmts } } diff --git a/src/libsyntax/ext/tt/quoted.rs b/src/libsyntax/ext/tt/quoted.rs index 1dab029785481..b142f09cdbc74 100644 --- a/src/libsyntax/ext/tt/quoted.rs +++ b/src/libsyntax/ext/tt/quoted.rs @@ -280,17 +280,17 @@ where // `tree` is a `$` token. Look at the next token in `trees` tokenstream::TokenTree::Token(span, token::Dollar) => match trees.next() { // `tree` is followed by a delimited set of token trees. This indicates the beginning - // of a repetition sequence in the macro (e.g., `$(pat)*`). - Some(tokenstream::TokenTree::Delimited(span, delimited)) => { + // of a repetition sequence in the macro (e.g. `$(pat)*`). + Some(tokenstream::TokenTree::Delimited(span, delim, tts)) => { // Must have `(` not `{` or `[` - if delimited.delim != token::Paren { - let tok = pprust::token_to_string(&token::OpenDelim(delimited.delim)); + if delim != token::Paren { + let tok = pprust::token_to_string(&token::OpenDelim(delim)); let msg = format!("expected `(`, found `{}`", tok); sess.span_diagnostic.span_err(span.entire(), &msg); } // Parse the contents of the sequence itself let sequence = parse( - delimited.tts.into(), + tts.into(), expect_matchers, sess, features, @@ -354,12 +354,12 @@ where // `tree` is the beginning of a delimited set of tokens (e.g., `(` or `{`). We need to // descend into the delimited set and further parse it. - tokenstream::TokenTree::Delimited(span, delimited) => TokenTree::Delimited( + tokenstream::TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited( span, Lrc::new(Delimited { - delim: delimited.delim, + delim: delim, tts: parse( - delimited.tts.into(), + tts.into(), expect_matchers, sess, features, diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs index 3d897d17e0b5c..a76779ffebdc0 100644 --- a/src/libsyntax/ext/tt/transcribe.rs +++ b/src/libsyntax/ext/tt/transcribe.rs @@ -17,7 +17,7 @@ use fold::noop_fold_tt; use parse::token::{self, Token, NtTT}; use smallvec::SmallVec; use syntax_pos::DUMMY_SP; -use tokenstream::{TokenStream, TokenTree, Delimited, DelimSpan}; +use tokenstream::{TokenStream, TokenTree, DelimSpan}; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sync::Lrc; @@ -105,10 +105,11 @@ pub fn transcribe(cx: &ExtCtxt, if result_stack.is_empty() { return TokenStream::concat(result); } - let tree = TokenTree::Delimited(span, Delimited { - delim: forest.delim, - tts: TokenStream::concat(result).into(), - }); + let tree = TokenTree::Delimited( + span, + forest.delim, + TokenStream::concat(result).into(), + ); result = result_stack.pop().unwrap(); result.push(tree.into()); } diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs index 82f52931cad7c..ecb0245263853 100644 --- a/src/libsyntax/fold.rs +++ b/src/libsyntax/fold.rs @@ -605,12 +605,10 @@ pub fn noop_fold_tt(tt: TokenTree, fld: &mut T) -> TokenTree { match tt { TokenTree::Token(span, tok) => TokenTree::Token(fld.new_span(span), fld.fold_token(tok)), - TokenTree::Delimited(span, delimed) => TokenTree::Delimited( + TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited( DelimSpan::from_pair(fld.new_span(span.open), fld.new_span(span.close)), - Delimited { - tts: fld.fold_tts(delimed.stream()).into(), - delim: delimed.delim, - } + delim, + fld.fold_tts(tts.stream()).into(), ), } } diff --git a/src/libsyntax/parse/lexer/tokentrees.rs b/src/libsyntax/parse/lexer/tokentrees.rs index 8047ab0146514..86c87cf898d09 100644 --- a/src/libsyntax/parse/lexer/tokentrees.rs +++ b/src/libsyntax/parse/lexer/tokentrees.rs @@ -11,7 +11,7 @@ use print::pprust::token_to_string; use parse::lexer::StringReader; use parse::{token, PResult}; -use tokenstream::{Delimited, DelimSpan, TokenStream, TokenTree}; +use tokenstream::{DelimSpan, TokenStream, TokenTree}; impl<'a> StringReader<'a> { // Parse a stream of tokens into a list of `TokenTree`s, up to an `Eof`. @@ -155,10 +155,11 @@ impl<'a> StringReader<'a> { _ => {} } - Ok(TokenTree::Delimited(delim_span, Delimited { + Ok(TokenTree::Delimited( + delim_span, delim, - tts: tts.into(), - }).into()) + tts.into(), + ).into()) }, token::CloseDelim(_) => { // An unexpected closing delimiter (i.e., there is no diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index 866dba24dcb0a..eb71003d3d0cf 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -764,7 +764,7 @@ mod tests { use attr::first_attr_value_str_by_name; use parse; use print::pprust::item_to_string; - use tokenstream::{self, DelimSpan, TokenTree}; + use tokenstream::{DelimSpan, TokenTree}; use util::parser_testing::string_to_stream; use util::parser_testing::{string_to_expr, string_to_item}; use with_globals; @@ -795,42 +795,41 @@ mod tests { Some(&TokenTree::Token(_, token::Ident(name_macro_rules, false))), Some(&TokenTree::Token(_, token::Not)), Some(&TokenTree::Token(_, token::Ident(name_zip, false))), - Some(&TokenTree::Delimited(_, ref macro_delimed)), + Some(&TokenTree::Delimited(_, macro_delim, ref macro_tts)), ) if name_macro_rules.name == "macro_rules" && name_zip.name == "zip" => { - let tts = ¯o_delimed.stream().trees().collect::>(); + let tts = ¯o_tts.stream().trees().collect::>(); match (tts.len(), tts.get(0), tts.get(1), tts.get(2)) { ( 3, - Some(&TokenTree::Delimited(_, ref first_delimed)), + Some(&TokenTree::Delimited(_, first_delim, ref first_tts)), Some(&TokenTree::Token(_, token::FatArrow)), - Some(&TokenTree::Delimited(_, ref second_delimed)), + Some(&TokenTree::Delimited(_, second_delim, ref second_tts)), ) - if macro_delimed.delim == token::Paren => { - let tts = &first_delimed.stream().trees().collect::>(); + if macro_delim == token::Paren => { + let tts = &first_tts.stream().trees().collect::>(); match (tts.len(), tts.get(0), tts.get(1)) { ( 2, Some(&TokenTree::Token(_, token::Dollar)), Some(&TokenTree::Token(_, token::Ident(ident, false))), ) - if first_delimed.delim == token::Paren && ident.name == "a" => {}, - _ => panic!("value 3: {:?}", *first_delimed), + if first_delim == token::Paren && ident.name == "a" => {}, + _ => panic!("value 3: {:?} {:?}", first_delim, first_tts), } - let tts = &second_delimed.stream().trees().collect::>(); + let tts = &second_tts.stream().trees().collect::>(); match (tts.len(), tts.get(0), tts.get(1)) { ( 2, Some(&TokenTree::Token(_, token::Dollar)), Some(&TokenTree::Token(_, token::Ident(ident, false))), ) - if second_delimed.delim == token::Paren - && ident.name == "a" => {}, - _ => panic!("value 4: {:?}", *second_delimed), + if second_delim == token::Paren && ident.name == "a" => {}, + _ => panic!("value 4: {:?} {:?}", second_delim, second_tts), } }, - _ => panic!("value 2: {:?}", *macro_delimed), + _ => panic!("value 2: {:?} {:?}", macro_delim, macro_tts), } }, _ => panic!("value: {:?}",tts), @@ -848,26 +847,24 @@ mod tests { TokenTree::Token(sp(3, 4), token::Ident(Ident::from_str("a"), false)).into(), TokenTree::Delimited( DelimSpan::from_pair(sp(5, 6), sp(13, 14)), - tokenstream::Delimited { - delim: token::DelimToken::Paren, - tts: TokenStream::concat(vec![ - TokenTree::Token(sp(6, 7), - token::Ident(Ident::from_str("b"), false)).into(), - TokenTree::Token(sp(8, 9), token::Colon).into(), - TokenTree::Token(sp(10, 13), - token::Ident(Ident::from_str("i32"), false)).into(), - ]).into(), - }).into(), + token::DelimToken::Paren, + TokenStream::concat(vec![ + TokenTree::Token(sp(6, 7), + token::Ident(Ident::from_str("b"), false)).into(), + TokenTree::Token(sp(8, 9), token::Colon).into(), + TokenTree::Token(sp(10, 13), + token::Ident(Ident::from_str("i32"), false)).into(), + ]).into(), + ).into(), TokenTree::Delimited( DelimSpan::from_pair(sp(15, 16), sp(20, 21)), - tokenstream::Delimited { - delim: token::DelimToken::Brace, - tts: TokenStream::concat(vec![ - TokenTree::Token(sp(17, 18), - token::Ident(Ident::from_str("b"), false)).into(), - TokenTree::Token(sp(18, 19), token::Semi).into(), - ]).into(), - }).into() + token::DelimToken::Brace, + TokenStream::concat(vec![ + TokenTree::Token(sp(17, 18), + token::Ident(Ident::from_str("b"), false)).into(), + TokenTree::Token(sp(18, 19), token::Semi).into(), + ]).into(), + ).into() ]); assert_eq!(tts, expected); diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index e3a3454de497f..ded6da9f3adb8 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -48,13 +48,14 @@ use errors::{self, Applicability, DiagnosticBuilder, DiagnosticId}; use parse::{self, SeqSep, classify, token}; use parse::lexer::TokenAndSpan; use parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration}; +use parse::token::DelimToken; use parse::{new_sub_parser_from_file, ParseSess, Directory, DirectoryOwnership}; use util::parser::{AssocOp, Fixity}; use print::pprust; use ptr::P; use parse::PResult; use ThinVec; -use tokenstream::{self, Delimited, DelimSpan, ThinTokenStream, TokenTree, TokenStream}; +use tokenstream::{self, DelimSpan, ThinTokenStream, TokenTree, TokenStream}; use symbol::{Symbol, keywords}; use std::borrow::Cow; @@ -293,13 +294,13 @@ enum LastToken { } impl TokenCursorFrame { - fn new(sp: DelimSpan, delimited: &Delimited) -> Self { + fn new(sp: DelimSpan, delim: DelimToken, tts: &ThinTokenStream) -> Self { TokenCursorFrame { - delim: delimited.delim, + delim: delim, span: sp, - open_delim: delimited.delim == token::NoDelim, - tree_cursor: delimited.stream().into_trees(), - close_delim: delimited.delim == token::NoDelim, + open_delim: delim == token::NoDelim, + tree_cursor: tts.stream().into_trees(), + close_delim: delim == token::NoDelim, last_token: LastToken::Was(None), } } @@ -310,14 +311,12 @@ impl TokenCursor { loop { let tree = if !self.frame.open_delim { self.frame.open_delim = true; - Delimited { delim: self.frame.delim, tts: TokenStream::empty().into() } - .open_tt(self.frame.span.open) + TokenTree::open_tt(self.frame.span.open, self.frame.delim) } else if let Some(tree) = self.frame.tree_cursor.next() { tree } else if !self.frame.close_delim { self.frame.close_delim = true; - Delimited { delim: self.frame.delim, tts: TokenStream::empty().into() } - .close_tt(self.frame.span.close) + TokenTree::close_tt(self.frame.span.close, self.frame.delim) } else if let Some(frame) = self.stack.pop() { self.frame = frame; continue @@ -332,8 +331,8 @@ impl TokenCursor { match tree { TokenTree::Token(sp, tok) => return TokenAndSpan { tok: tok, sp: sp }, - TokenTree::Delimited(sp, ref delimited) => { - let frame = TokenCursorFrame::new(sp, delimited); + TokenTree::Delimited(sp, delim, tts) => { + let frame = TokenCursorFrame::new(sp, delim, &tts); self.stack.push(mem::replace(&mut self.frame, frame)); } } @@ -362,25 +361,28 @@ impl TokenCursor { } let delim_span = DelimSpan::from_single(sp); - let body = TokenTree::Delimited(delim_span, Delimited { - delim: token::Bracket, - tts: [TokenTree::Token(sp, token::Ident(ast::Ident::from_str("doc"), false)), - TokenTree::Token(sp, token::Eq), - TokenTree::Token(sp, token::Literal( - token::StrRaw(Symbol::intern(&stripped), num_of_hashes), None))] - .iter().cloned().collect::().into(), - }); + let body = TokenTree::Delimited( + delim_span, + token::Bracket, + [TokenTree::Token(sp, token::Ident(ast::Ident::from_str("doc"), false)), + TokenTree::Token(sp, token::Eq), + TokenTree::Token(sp, token::Literal( + token::StrRaw(Symbol::intern(&stripped), num_of_hashes), None)) + ] + .iter().cloned().collect::().into(), + ); - self.stack.push(mem::replace(&mut self.frame, TokenCursorFrame::new(delim_span, &Delimited { - delim: token::NoDelim, - tts: if doc_comment_style(&name.as_str()) == AttrStyle::Inner { + self.stack.push(mem::replace(&mut self.frame, TokenCursorFrame::new( + delim_span, + token::NoDelim, + &if doc_comment_style(&name.as_str()) == AttrStyle::Inner { [TokenTree::Token(sp, token::Pound), TokenTree::Token(sp, token::Not), body] .iter().cloned().collect::().into() } else { [TokenTree::Token(sp, token::Pound), body] .iter().cloned().collect::().into() }, - }))); + ))); self.next() } @@ -561,10 +563,11 @@ impl<'a> Parser<'a> { root_module_name: None, expected_tokens: Vec::new(), token_cursor: TokenCursor { - frame: TokenCursorFrame::new(DelimSpan::dummy(), &Delimited { - delim: token::NoDelim, - tts: tokens.into(), - }), + frame: TokenCursorFrame::new( + DelimSpan::dummy(), + token::NoDelim, + &tokens.into(), + ), stack: Vec::new(), }, desugar_doc_comments, @@ -1238,7 +1241,7 @@ impl<'a> Parser<'a> { f(&match self.token_cursor.frame.tree_cursor.look_ahead(dist - 1) { Some(tree) => match tree { TokenTree::Token(_, tok) => tok, - TokenTree::Delimited(_, delimited) => token::OpenDelim(delimited.delim), + TokenTree::Delimited(_, delim, _) => token::OpenDelim(delim), }, None => token::CloseDelim(self.token_cursor.frame.delim), }) @@ -1251,7 +1254,7 @@ impl<'a> Parser<'a> { match self.token_cursor.frame.tree_cursor.look_ahead(dist - 1) { Some(TokenTree::Token(span, _)) => span, - Some(TokenTree::Delimited(span, _)) => span.entire(), + Some(TokenTree::Delimited(span, ..)) => span.entire(), None => self.look_ahead_span(dist - 1), } } @@ -2317,8 +2320,8 @@ impl<'a> Parser<'a> { return Err(err) } }; - let delimited = match self.parse_token_tree() { - TokenTree::Delimited(_, delimited) => delimited, + let tts = match self.parse_token_tree() { + TokenTree::Delimited(_, _, tts) => tts, _ => unreachable!(), }; let delim = match delim { @@ -2327,7 +2330,7 @@ impl<'a> Parser<'a> { token::Brace => MacDelimiter::Brace, token::NoDelim => self.bug("unexpected no delimiter"), }; - Ok((delim, delimited.stream().into())) + Ok((delim, tts.stream().into())) } /// At the bottom (top?) of the precedence hierarchy, @@ -2892,10 +2895,11 @@ impl<'a> Parser<'a> { self.token_cursor.stack.pop().unwrap()); self.span = frame.span.entire(); self.bump(); - TokenTree::Delimited(frame.span, Delimited { - delim: frame.delim, - tts: frame.tree_cursor.original_stream().into(), - }) + TokenTree::Delimited( + frame.span, + frame.delim, + frame.tree_cursor.original_stream().into(), + ) }, token::CloseDelim(_) | token::Eof => unreachable!(), _ => { @@ -4609,7 +4613,7 @@ impl<'a> Parser<'a> { let ident = self.parse_ident()?; let tokens = if self.check(&token::OpenDelim(token::Brace)) { match self.parse_token_tree() { - TokenTree::Delimited(_, ref delimited) => delimited.stream(), + TokenTree::Delimited(_, _, tts) => tts.stream(), _ => unreachable!(), } } else if self.check(&token::OpenDelim(token::Paren)) { diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 6b9cc2f9792e3..8e4d3c0166bb2 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -818,16 +818,13 @@ fn prepend_attrs(sess: &ParseSess, brackets.push(attr.tokens.clone()); - let tokens = tokenstream::Delimited { - delim: DelimToken::Bracket, - tts: brackets.build().into(), - }; // The span we list here for `#` and for `[ ... ]` are both wrong in // that it encompasses more than each token, but it hopefully is "good // enough" for now at least. builder.push(tokenstream::TokenTree::Token(attr.span, Pound)); let delim_span = DelimSpan::from_single(attr.span); - builder.push(tokenstream::TokenTree::Delimited(delim_span, tokens)); + builder.push(tokenstream::TokenTree::Delimited( + delim_span, DelimToken::Bracket, brackets.build().into())); } builder.push(tokens.clone()); Some(builder.build()) diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs index e50f28897dd12..41165c7e36d58 100644 --- a/src/libsyntax/print/pprust.rs +++ b/src/libsyntax/print/pprust.rs @@ -815,12 +815,12 @@ pub trait PrintState<'a> { _ => Ok(()) } } - TokenTree::Delimited(_, ref delimed) => { - self.writer().word(token_to_string(&delimed.open_token()))?; + TokenTree::Delimited(_, delim, tts) => { + self.writer().word(token_to_string(&token::OpenDelim(delim)))?; self.writer().space()?; - self.print_tts(delimed.stream())?; + self.print_tts(tts.stream())?; self.writer().space()?; - self.writer().word(token_to_string(&delimed.close_token())) + self.writer().word(token_to_string(&token::CloseDelim(delim))) }, } } diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index 242299f1b1f6f..90191c54126d1 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -34,52 +34,6 @@ use util::RcVec; use std::borrow::Cow; use std::{fmt, iter, mem}; -/// A delimited sequence of token trees -#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug)] -pub struct Delimited { - /// The type of delimiter - pub delim: DelimToken, - /// The delimited sequence of token trees - pub tts: ThinTokenStream, -} - -impl Delimited { - /// Returns the opening delimiter as a token. - pub fn open_token(&self) -> token::Token { - token::OpenDelim(self.delim) - } - - /// Returns the closing delimiter as a token. - pub fn close_token(&self) -> token::Token { - token::CloseDelim(self.delim) - } - - /// Returns the opening delimiter as a token tree. - pub fn open_tt(&self, span: Span) -> TokenTree { - let open_span = if span.is_dummy() { - span - } else { - span.with_hi(span.lo() + BytePos(self.delim.len() as u32)) - }; - TokenTree::Token(open_span, self.open_token()) - } - - /// Returns the closing delimiter as a token tree. - pub fn close_tt(&self, span: Span) -> TokenTree { - let close_span = if span.is_dummy() { - span - } else { - span.with_lo(span.hi() - BytePos(self.delim.len() as u32)) - }; - TokenTree::Token(close_span, self.close_token()) - } - - /// Returns the token trees inside the delimiters. - pub fn stream(&self) -> TokenStream { - self.tts.clone().into() - } -} - /// When the main rust parser encounters a syntax-extension invocation, it /// parses the arguments to the invocation as a token-tree. This is a very /// loose structure, such that all sorts of different AST-fragments can @@ -97,7 +51,7 @@ pub enum TokenTree { /// A single token Token(Span, token::Token), /// A delimited sequence of token trees - Delimited(DelimSpan, Delimited), + Delimited(DelimSpan, DelimToken, ThinTokenStream), } impl TokenTree { @@ -116,9 +70,10 @@ impl TokenTree { pub fn eq_unspanned(&self, other: &TokenTree) -> bool { match (self, other) { (&TokenTree::Token(_, ref tk), &TokenTree::Token(_, ref tk2)) => tk == tk2, - (&TokenTree::Delimited(_, ref dl), &TokenTree::Delimited(_, ref dl2)) => { - dl.delim == dl2.delim && - dl.stream().eq_unspanned(&dl2.stream()) + (&TokenTree::Delimited(_, delim, ref tts), + &TokenTree::Delimited(_, delim2, ref tts2)) => { + delim == delim2 && + tts.stream().eq_unspanned(&tts2.stream()) } (_, _) => false, } @@ -134,9 +89,10 @@ impl TokenTree { (&TokenTree::Token(_, ref tk), &TokenTree::Token(_, ref tk2)) => { tk.probably_equal_for_proc_macro(tk2) } - (&TokenTree::Delimited(_, ref dl), &TokenTree::Delimited(_, ref dl2)) => { - dl.delim == dl2.delim && - dl.stream().probably_equal_for_proc_macro(&dl2.stream()) + (&TokenTree::Delimited(_, delim, ref tts), + &TokenTree::Delimited(_, delim2, ref tts2)) => { + delim == delim2 && + tts.stream().probably_equal_for_proc_macro(&tts2.stream()) } (_, _) => false, } @@ -146,7 +102,7 @@ impl TokenTree { pub fn span(&self) -> Span { match *self { TokenTree::Token(sp, _) => sp, - TokenTree::Delimited(sp, _) => sp.entire(), + TokenTree::Delimited(sp, ..) => sp.entire(), } } @@ -154,7 +110,7 @@ impl TokenTree { pub fn set_span(&mut self, span: Span) { match *self { TokenTree::Token(ref mut sp, _) => *sp = span, - TokenTree::Delimited(ref mut sp, _) => *sp = DelimSpan::from_single(span), + TokenTree::Delimited(ref mut sp, ..) => *sp = DelimSpan::from_single(span), } } @@ -169,6 +125,26 @@ impl TokenTree { pub fn joint(self) -> TokenStream { TokenStream { kind: TokenStreamKind::JointTree(self) } } + + /// Returns the opening delimiter as a token tree. + pub fn open_tt(span: Span, delim: DelimToken) -> TokenTree { + let open_span = if span.is_dummy() { + span + } else { + span.with_hi(span.lo() + BytePos(delim.len() as u32)) + }; + TokenTree::Token(open_span, token::OpenDelim(delim)) + } + + /// Returns the closing delimiter as a token tree. + pub fn close_tt(span: Span, delim: DelimToken) -> TokenTree { + let close_span = if span.is_dummy() { + span + } else { + span.with_lo(span.hi() - BytePos(delim.len() as u32)) + }; + TokenTree::Token(close_span, token::CloseDelim(delim)) + } } /// # Token Streams @@ -182,6 +158,10 @@ pub struct TokenStream { kind: TokenStreamKind, } +// `TokenStream` is used a lot. Make sure it doesn't unintentionally get bigger. +#[cfg(target_arch = "x86_64")] +static_assert!(MEM_SIZE_OF_TOKEN_STREAM: mem::size_of::() == 40); + impl TokenStream { /// Given a `TokenStream` with a `Stream` of only two arguments, return a new `TokenStream` /// separating the two arguments with a comma for diagnostic suggestions. @@ -198,7 +178,7 @@ impl TokenStream { continue; } (TokenStreamKind::Tree(TokenTree::Token(sp, _)), _) => *sp, - (TokenStreamKind::Tree(TokenTree::Delimited(sp, _)), _) => sp.entire(), + (TokenStreamKind::Tree(TokenTree::Delimited(sp, ..)), _) => sp.entire(), _ => continue, }; let sp = sp.shrink_to_hi(); @@ -678,6 +658,12 @@ impl Cursor { #[derive(Debug, Clone)] pub struct ThinTokenStream(Option>); +impl ThinTokenStream { + pub fn stream(&self) -> TokenStream { + self.clone().into() + } +} + impl From for ThinTokenStream { fn from(stream: TokenStream) -> ThinTokenStream { ThinTokenStream(match stream.kind { diff --git a/src/libsyntax/visit.rs b/src/libsyntax/visit.rs index 4be831a0b3f6b..6747598f3753f 100644 --- a/src/libsyntax/visit.rs +++ b/src/libsyntax/visit.rs @@ -841,7 +841,7 @@ pub fn walk_attribute<'a, V: Visitor<'a>>(visitor: &mut V, attr: &'a Attribute) pub fn walk_tt<'a, V: Visitor<'a>>(visitor: &mut V, tt: TokenTree) { match tt { TokenTree::Token(_, tok) => visitor.visit_token(tok), - TokenTree::Delimited(_, delimed) => visitor.visit_tts(delimed.stream()), + TokenTree::Delimited(_, _, tts) => visitor.visit_tts(tts.stream()), } } diff --git a/src/libsyntax_ext/proc_macro_server.rs b/src/libsyntax_ext/proc_macro_server.rs index 6c7da589a4257..a04d6c92b7817 100644 --- a/src/libsyntax_ext/proc_macro_server.rs +++ b/src/libsyntax_ext/proc_macro_server.rs @@ -64,11 +64,11 @@ impl FromInternal<(TokenStream, &'_ ParseSess, &'_ mut Vec)> let (tree, joint) = stream.as_tree(); let (span, token) = match tree { - tokenstream::TokenTree::Delimited(span, delimed) => { - let delimiter = Delimiter::from_internal(delimed.delim); + tokenstream::TokenTree::Delimited(span, delim, tts) => { + let delimiter = Delimiter::from_internal(delim); return TokenTree::Group(Group { delimiter, - stream: delimed.tts.into(), + stream: tts.into(), span, }); } @@ -232,10 +232,8 @@ impl ToInternal for TokenTree { }) => { return tokenstream::TokenTree::Delimited( span, - tokenstream::Delimited { - delim: delimiter.to_internal(), - tts: stream.into(), - }, + delimiter.to_internal(), + stream.into(), ) .into(); }