Skip to content

Commit

Permalink
Remove tokenstream::Delimited.
Browse files Browse the repository at this point in the history
Because it's an extra type layer that doesn't really help; in a couple
of places it actively gets in the way, and overall removing it makes the
code nicer. It does, however, move `tokenstream::TokenTree` further away
from the `TokenTree` in `quote.rs`.

More importantly, this change reduces the size of `TokenStream` from 48
bytes to 40 bytes on x86-64, which is enough to slightly reduce
instruction counts on numerous benchmarks, the best by 1.5%.

Note that `open_tt` and `close_tt` have gone from being methods on
`Delimited` to associated methods of `TokenTree`.
  • Loading branch information
nnethercote committed Dec 10, 2018
1 parent b755501 commit 1fe2c03
Show file tree
Hide file tree
Showing 18 changed files with 182 additions and 203 deletions.
10 changes: 4 additions & 6 deletions src/librustc/hir/lowering.rs
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ use syntax::ptr::P;
use syntax::source_map::{self, respan, CompilerDesugaringKind, Spanned};
use syntax::std_inject;
use syntax::symbol::{keywords, Symbol};
use syntax::tokenstream::{Delimited, TokenStream, TokenTree};
use syntax::tokenstream::{TokenStream, TokenTree};
use syntax::parse::token::Token;
use syntax::visit::{self, Visitor};
use syntax_pos::{Span, MultiSpan};
Expand Down Expand Up @@ -1088,12 +1088,10 @@ impl<'a> LoweringContext<'a> {
fn lower_token_tree(&mut self, tree: TokenTree) -> TokenStream {
match tree {
TokenTree::Token(span, token) => self.lower_token(token, span),
TokenTree::Delimited(span, delimited) => TokenTree::Delimited(
TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited(
span,
Delimited {
delim: delimited.delim,
tts: self.lower_token_stream(delimited.tts.into()).into(),
},
delim,
self.lower_token_stream(tts.into()).into(),
).into(),
}
}
Expand Down
6 changes: 3 additions & 3 deletions src/librustc/ich/impls_syntax.rs
Original file line number Diff line number Diff line change
Expand Up @@ -265,10 +265,10 @@ for tokenstream::TokenTree {
span.hash_stable(hcx, hasher);
hash_token(token, hcx, hasher);
}
tokenstream::TokenTree::Delimited(span, ref delimited) => {
tokenstream::TokenTree::Delimited(span, delim, ref tts) => {
span.hash_stable(hcx, hasher);
std_hash::Hash::hash(&delimited.delim, hasher);
for sub_tt in delimited.stream().trees() {
std_hash::Hash::hash(&delim, hasher);
for sub_tt in tts.stream().trees() {
sub_tt.hash_stable(hcx, hasher);
}
}
Expand Down
4 changes: 2 additions & 2 deletions src/librustc_lint/builtin.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1540,8 +1540,8 @@ impl KeywordIdents {
}
_ => {},
}
TokenTree::Delimited(_, ref delim) => {
self.check_tokens(cx, delim.tts.clone().into())
TokenTree::Delimited(_, _, tts) => {
self.check_tokens(cx, tts.stream())
},
}
}
Expand Down
2 changes: 1 addition & 1 deletion src/libsyntax/ast.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1235,7 +1235,7 @@ pub enum MacDelimiter {

impl Mac_ {
pub fn stream(&self) -> TokenStream {
self.tts.clone().into()
self.tts.stream()
}
}

Expand Down
15 changes: 8 additions & 7 deletions src/libsyntax/attr/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ use parse::token::{self, Token};
use ptr::P;
use symbol::Symbol;
use ThinVec;
use tokenstream::{TokenStream, TokenTree, Delimited, DelimSpan};
use tokenstream::{TokenStream, TokenTree, DelimSpan};
use GLOBALS;

use std::iter;
Expand Down Expand Up @@ -549,10 +549,11 @@ impl MetaItemKind {
}
tokens.push(item.node.tokens());
}
TokenTree::Delimited(DelimSpan::from_single(span), Delimited {
delim: token::Paren,
tts: TokenStream::concat(tokens).into(),
}).into()
TokenTree::Delimited(
DelimSpan::from_single(span),
token::Paren,
TokenStream::concat(tokens).into(),
).into()
}
}
}
Expand All @@ -570,9 +571,9 @@ impl MetaItemKind {
None
};
}
Some(TokenTree::Delimited(_, ref delimited)) if delimited.delim == token::Paren => {
Some(TokenTree::Delimited(_, delim, ref tts)) if delim == token::Paren => {
tokens.next();
delimited.stream()
tts.stream()
}
_ => return Some(MetaItemKind::Word),
};
Expand Down
4 changes: 2 additions & 2 deletions src/libsyntax/ext/expand.rs
Original file line number Diff line number Diff line change
Expand Up @@ -622,9 +622,9 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
fn extract_proc_macro_attr_input(&self, tokens: TokenStream, span: Span) -> TokenStream {
let mut trees = tokens.trees();
match trees.next() {
Some(TokenTree::Delimited(_, delim)) => {
Some(TokenTree::Delimited(_, _, tts)) => {
if trees.next().is_none() {
return delim.tts.into()
return tts.into()
}
}
Some(TokenTree::Token(..)) => {}
Expand Down
32 changes: 15 additions & 17 deletions src/libsyntax/ext/quote.rs
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ pub mod rt {
use symbol::Symbol;
use ThinVec;

use tokenstream::{self, DelimSpan, TokenTree, TokenStream};
use tokenstream::{DelimSpan, TokenTree, TokenStream};

pub use parse::new_parser_from_tts;
pub use syntax_pos::{BytePos, Span, DUMMY_SP, FileName};
Expand Down Expand Up @@ -246,9 +246,9 @@ pub mod rt {
inner.push(self.tokens.clone());

let delim_span = DelimSpan::from_single(self.span);
r.push(TokenTree::Delimited(delim_span, tokenstream::Delimited {
delim: token::Bracket, tts: TokenStream::concat(inner).into()
}));
r.push(TokenTree::Delimited(
delim_span, token::Bracket, TokenStream::concat(inner).into()
));
r
}
}
Expand All @@ -262,10 +262,9 @@ pub mod rt {

impl ToTokens for () {
fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
vec![TokenTree::Delimited(DelimSpan::dummy(), tokenstream::Delimited {
delim: token::Paren,
tts: TokenStream::empty().into(),
})]
vec![
TokenTree::Delimited(DelimSpan::dummy(), token::Paren, TokenStream::empty().into())
]
}
}

Expand Down Expand Up @@ -382,8 +381,6 @@ pub mod rt {

// Replaces `Token::OpenDelim .. Token::CloseDelim` with `TokenTree::Delimited(..)`.
pub fn unflatten(tts: Vec<TokenTree>) -> Vec<TokenTree> {
use tokenstream::Delimited;

let mut results = Vec::new();
let mut result = Vec::new();
let mut open_span = DUMMY_SP;
Expand All @@ -395,10 +392,11 @@ pub fn unflatten(tts: Vec<TokenTree>) -> Vec<TokenTree> {
}
TokenTree::Token(span, token::CloseDelim(delim)) => {
let delim_span = DelimSpan::from_pair(open_span, span);
let tree = TokenTree::Delimited(delim_span, Delimited {
let tree = TokenTree::Delimited(
delim_span,
delim,
tts: result.into_iter().map(TokenStream::from).collect::<TokenStream>().into(),
});
result.into_iter().map(TokenStream::from).collect::<TokenStream>().into(),
);
result = results.pop().unwrap();
result.push(tree);
}
Expand Down Expand Up @@ -758,10 +756,10 @@ fn statements_mk_tt(cx: &ExtCtxt, tt: &TokenTree, quoted: bool) -> Vec<ast::Stmt
vec![e_tok]);
vec![cx.stmt_expr(e_push)]
},
TokenTree::Delimited(span, ref delimed) => {
let mut stmts = statements_mk_tt(cx, &delimed.open_tt(span.open), false);
stmts.extend(statements_mk_tts(cx, delimed.stream()));
stmts.extend(statements_mk_tt(cx, &delimed.close_tt(span.close), false));
TokenTree::Delimited(span, delim, ref tts) => {
let mut stmts = statements_mk_tt(cx, &TokenTree::open_tt(span.open, delim), false);
stmts.extend(statements_mk_tts(cx, tts.stream()));
stmts.extend(statements_mk_tt(cx, &TokenTree::close_tt(span.close, delim), false));
stmts
}
}
Expand Down
16 changes: 8 additions & 8 deletions src/libsyntax/ext/tt/quoted.rs
Original file line number Diff line number Diff line change
Expand Up @@ -280,17 +280,17 @@ where
// `tree` is a `$` token. Look at the next token in `trees`
tokenstream::TokenTree::Token(span, token::Dollar) => match trees.next() {
// `tree` is followed by a delimited set of token trees. This indicates the beginning
// of a repetition sequence in the macro (e.g., `$(pat)*`).
Some(tokenstream::TokenTree::Delimited(span, delimited)) => {
// of a repetition sequence in the macro (e.g. `$(pat)*`).
Some(tokenstream::TokenTree::Delimited(span, delim, tts)) => {
// Must have `(` not `{` or `[`
if delimited.delim != token::Paren {
let tok = pprust::token_to_string(&token::OpenDelim(delimited.delim));
if delim != token::Paren {
let tok = pprust::token_to_string(&token::OpenDelim(delim));
let msg = format!("expected `(`, found `{}`", tok);
sess.span_diagnostic.span_err(span.entire(), &msg);
}
// Parse the contents of the sequence itself
let sequence = parse(
delimited.tts.into(),
tts.into(),
expect_matchers,
sess,
features,
Expand Down Expand Up @@ -354,12 +354,12 @@ where

// `tree` is the beginning of a delimited set of tokens (e.g., `(` or `{`). We need to
// descend into the delimited set and further parse it.
tokenstream::TokenTree::Delimited(span, delimited) => TokenTree::Delimited(
tokenstream::TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited(
span,
Lrc::new(Delimited {
delim: delimited.delim,
delim: delim,
tts: parse(
delimited.tts.into(),
tts.into(),
expect_matchers,
sess,
features,
Expand Down
11 changes: 6 additions & 5 deletions src/libsyntax/ext/tt/transcribe.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ use fold::noop_fold_tt;
use parse::token::{self, Token, NtTT};
use smallvec::SmallVec;
use syntax_pos::DUMMY_SP;
use tokenstream::{TokenStream, TokenTree, Delimited, DelimSpan};
use tokenstream::{TokenStream, TokenTree, DelimSpan};

use rustc_data_structures::fx::FxHashMap;
use rustc_data_structures::sync::Lrc;
Expand Down Expand Up @@ -105,10 +105,11 @@ pub fn transcribe(cx: &ExtCtxt,
if result_stack.is_empty() {
return TokenStream::concat(result);
}
let tree = TokenTree::Delimited(span, Delimited {
delim: forest.delim,
tts: TokenStream::concat(result).into(),
});
let tree = TokenTree::Delimited(
span,
forest.delim,
TokenStream::concat(result).into(),
);
result = result_stack.pop().unwrap();
result.push(tree.into());
}
Expand Down
8 changes: 3 additions & 5 deletions src/libsyntax/fold.rs
Original file line number Diff line number Diff line change
Expand Up @@ -605,12 +605,10 @@ pub fn noop_fold_tt<T: Folder>(tt: TokenTree, fld: &mut T) -> TokenTree {
match tt {
TokenTree::Token(span, tok) =>
TokenTree::Token(fld.new_span(span), fld.fold_token(tok)),
TokenTree::Delimited(span, delimed) => TokenTree::Delimited(
TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited(
DelimSpan::from_pair(fld.new_span(span.open), fld.new_span(span.close)),
Delimited {
tts: fld.fold_tts(delimed.stream()).into(),
delim: delimed.delim,
}
delim,
fld.fold_tts(tts.stream()).into(),
),
}
}
Expand Down
9 changes: 5 additions & 4 deletions src/libsyntax/parse/lexer/tokentrees.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
use print::pprust::token_to_string;
use parse::lexer::StringReader;
use parse::{token, PResult};
use tokenstream::{Delimited, DelimSpan, TokenStream, TokenTree};
use tokenstream::{DelimSpan, TokenStream, TokenTree};

impl<'a> StringReader<'a> {
// Parse a stream of tokens into a list of `TokenTree`s, up to an `Eof`.
Expand Down Expand Up @@ -155,10 +155,11 @@ impl<'a> StringReader<'a> {
_ => {}
}

Ok(TokenTree::Delimited(delim_span, Delimited {
Ok(TokenTree::Delimited(
delim_span,
delim,
tts: tts.into(),
}).into())
tts.into(),
).into())
},
token::CloseDelim(_) => {
// An unexpected closing delimiter (i.e., there is no
Expand Down
61 changes: 29 additions & 32 deletions src/libsyntax/parse/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -764,7 +764,7 @@ mod tests {
use attr::first_attr_value_str_by_name;
use parse;
use print::pprust::item_to_string;
use tokenstream::{self, DelimSpan, TokenTree};
use tokenstream::{DelimSpan, TokenTree};
use util::parser_testing::string_to_stream;
use util::parser_testing::{string_to_expr, string_to_item};
use with_globals;
Expand Down Expand Up @@ -795,42 +795,41 @@ mod tests {
Some(&TokenTree::Token(_, token::Ident(name_macro_rules, false))),
Some(&TokenTree::Token(_, token::Not)),
Some(&TokenTree::Token(_, token::Ident(name_zip, false))),
Some(&TokenTree::Delimited(_, ref macro_delimed)),
Some(&TokenTree::Delimited(_, macro_delim, ref macro_tts)),
)
if name_macro_rules.name == "macro_rules"
&& name_zip.name == "zip" => {
let tts = &macro_delimed.stream().trees().collect::<Vec<_>>();
let tts = &macro_tts.stream().trees().collect::<Vec<_>>();
match (tts.len(), tts.get(0), tts.get(1), tts.get(2)) {
(
3,
Some(&TokenTree::Delimited(_, ref first_delimed)),
Some(&TokenTree::Delimited(_, first_delim, ref first_tts)),
Some(&TokenTree::Token(_, token::FatArrow)),
Some(&TokenTree::Delimited(_, ref second_delimed)),
Some(&TokenTree::Delimited(_, second_delim, ref second_tts)),
)
if macro_delimed.delim == token::Paren => {
let tts = &first_delimed.stream().trees().collect::<Vec<_>>();
if macro_delim == token::Paren => {
let tts = &first_tts.stream().trees().collect::<Vec<_>>();
match (tts.len(), tts.get(0), tts.get(1)) {
(
2,
Some(&TokenTree::Token(_, token::Dollar)),
Some(&TokenTree::Token(_, token::Ident(ident, false))),
)
if first_delimed.delim == token::Paren && ident.name == "a" => {},
_ => panic!("value 3: {:?}", *first_delimed),
if first_delim == token::Paren && ident.name == "a" => {},
_ => panic!("value 3: {:?} {:?}", first_delim, first_tts),
}
let tts = &second_delimed.stream().trees().collect::<Vec<_>>();
let tts = &second_tts.stream().trees().collect::<Vec<_>>();
match (tts.len(), tts.get(0), tts.get(1)) {
(
2,
Some(&TokenTree::Token(_, token::Dollar)),
Some(&TokenTree::Token(_, token::Ident(ident, false))),
)
if second_delimed.delim == token::Paren
&& ident.name == "a" => {},
_ => panic!("value 4: {:?}", *second_delimed),
if second_delim == token::Paren && ident.name == "a" => {},
_ => panic!("value 4: {:?} {:?}", second_delim, second_tts),
}
},
_ => panic!("value 2: {:?}", *macro_delimed),
_ => panic!("value 2: {:?} {:?}", macro_delim, macro_tts),
}
},
_ => panic!("value: {:?}",tts),
Expand All @@ -848,26 +847,24 @@ mod tests {
TokenTree::Token(sp(3, 4), token::Ident(Ident::from_str("a"), false)).into(),
TokenTree::Delimited(
DelimSpan::from_pair(sp(5, 6), sp(13, 14)),
tokenstream::Delimited {
delim: token::DelimToken::Paren,
tts: TokenStream::concat(vec![
TokenTree::Token(sp(6, 7),
token::Ident(Ident::from_str("b"), false)).into(),
TokenTree::Token(sp(8, 9), token::Colon).into(),
TokenTree::Token(sp(10, 13),
token::Ident(Ident::from_str("i32"), false)).into(),
]).into(),
}).into(),
token::DelimToken::Paren,
TokenStream::concat(vec![
TokenTree::Token(sp(6, 7),
token::Ident(Ident::from_str("b"), false)).into(),
TokenTree::Token(sp(8, 9), token::Colon).into(),
TokenTree::Token(sp(10, 13),
token::Ident(Ident::from_str("i32"), false)).into(),
]).into(),
).into(),
TokenTree::Delimited(
DelimSpan::from_pair(sp(15, 16), sp(20, 21)),
tokenstream::Delimited {
delim: token::DelimToken::Brace,
tts: TokenStream::concat(vec![
TokenTree::Token(sp(17, 18),
token::Ident(Ident::from_str("b"), false)).into(),
TokenTree::Token(sp(18, 19), token::Semi).into(),
]).into(),
}).into()
token::DelimToken::Brace,
TokenStream::concat(vec![
TokenTree::Token(sp(17, 18),
token::Ident(Ident::from_str("b"), false)).into(),
TokenTree::Token(sp(18, 19), token::Semi).into(),
]).into(),
).into()
]);

assert_eq!(tts, expected);
Expand Down
Loading

0 comments on commit 1fe2c03

Please sign in to comment.