Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Remove tokenstream::Delimited. #56369

Merged
merged 1 commit into from
Dec 10, 2018
Merged

Remove tokenstream::Delimited. #56369

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 4 additions & 6 deletions src/librustc/hir/lowering.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ use syntax::ptr::P;
use syntax::source_map::{self, respan, CompilerDesugaringKind, Spanned};
use syntax::std_inject;
use syntax::symbol::{keywords, Symbol};
use syntax::tokenstream::{Delimited, TokenStream, TokenTree};
use syntax::tokenstream::{TokenStream, TokenTree};
use syntax::parse::token::Token;
use syntax::visit::{self, Visitor};
use syntax_pos::{Span, MultiSpan};
Expand Down Expand Up @@ -1088,12 +1088,10 @@ impl<'a> LoweringContext<'a> {
fn lower_token_tree(&mut self, tree: TokenTree) -> TokenStream {
match tree {
TokenTree::Token(span, token) => self.lower_token(token, span),
TokenTree::Delimited(span, delimited) => TokenTree::Delimited(
TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited(
span,
Delimited {
delim: delimited.delim,
tts: self.lower_token_stream(delimited.tts.into()).into(),
},
delim,
self.lower_token_stream(tts.into()).into(),
).into(),
}
}
Expand Down
6 changes: 3 additions & 3 deletions src/librustc/ich/impls_syntax.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -265,10 +265,10 @@ for tokenstream::TokenTree {
span.hash_stable(hcx, hasher);
hash_token(token, hcx, hasher);
}
tokenstream::TokenTree::Delimited(span, ref delimited) => {
tokenstream::TokenTree::Delimited(span, delim, ref tts) => {
span.hash_stable(hcx, hasher);
std_hash::Hash::hash(&delimited.delim, hasher);
for sub_tt in delimited.stream().trees() {
std_hash::Hash::hash(&delim, hasher);
for sub_tt in tts.stream().trees() {
sub_tt.hash_stable(hcx, hasher);
}
}
Expand Down
4 changes: 2 additions & 2 deletions src/librustc_lint/builtin.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1540,8 +1540,8 @@ impl KeywordIdents {
}
_ => {},
}
TokenTree::Delimited(_, ref delim) => {
self.check_tokens(cx, delim.tts.clone().into())
TokenTree::Delimited(_, _, tts) => {
self.check_tokens(cx, tts.stream())
},
}
}
Expand Down
2 changes: 1 addition & 1 deletion src/libsyntax/ast.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1235,7 +1235,7 @@ pub enum MacDelimiter {

impl Mac_ {
pub fn stream(&self) -> TokenStream {
self.tts.clone().into()
self.tts.stream()
}
}

Expand Down
15 changes: 8 additions & 7 deletions src/libsyntax/attr/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ use parse::token::{self, Token};
use ptr::P;
use symbol::Symbol;
use ThinVec;
use tokenstream::{TokenStream, TokenTree, Delimited, DelimSpan};
use tokenstream::{TokenStream, TokenTree, DelimSpan};
use GLOBALS;

use std::iter;
Expand Down Expand Up @@ -549,10 +549,11 @@ impl MetaItemKind {
}
tokens.push(item.node.tokens());
}
TokenTree::Delimited(DelimSpan::from_single(span), Delimited {
delim: token::Paren,
tts: TokenStream::concat(tokens).into(),
}).into()
TokenTree::Delimited(
DelimSpan::from_single(span),
token::Paren,
TokenStream::concat(tokens).into(),
).into()
}
}
}
Expand All @@ -570,9 +571,9 @@ impl MetaItemKind {
None
};
}
Some(TokenTree::Delimited(_, ref delimited)) if delimited.delim == token::Paren => {
Some(TokenTree::Delimited(_, delim, ref tts)) if delim == token::Paren => {
tokens.next();
delimited.stream()
tts.stream()
}
_ => return Some(MetaItemKind::Word),
};
Expand Down
4 changes: 2 additions & 2 deletions src/libsyntax/ext/expand.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -622,9 +622,9 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
fn extract_proc_macro_attr_input(&self, tokens: TokenStream, span: Span) -> TokenStream {
let mut trees = tokens.trees();
match trees.next() {
Some(TokenTree::Delimited(_, delim)) => {
Some(TokenTree::Delimited(_, _, tts)) => {
if trees.next().is_none() {
return delim.tts.into()
return tts.into()
}
}
Some(TokenTree::Token(..)) => {}
Expand Down
32 changes: 15 additions & 17 deletions src/libsyntax/ext/quote.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ pub mod rt {
use symbol::Symbol;
use ThinVec;

use tokenstream::{self, DelimSpan, TokenTree, TokenStream};
use tokenstream::{DelimSpan, TokenTree, TokenStream};

pub use parse::new_parser_from_tts;
pub use syntax_pos::{BytePos, Span, DUMMY_SP, FileName};
Expand Down Expand Up @@ -246,9 +246,9 @@ pub mod rt {
inner.push(self.tokens.clone());

let delim_span = DelimSpan::from_single(self.span);
r.push(TokenTree::Delimited(delim_span, tokenstream::Delimited {
delim: token::Bracket, tts: TokenStream::concat(inner).into()
}));
r.push(TokenTree::Delimited(
delim_span, token::Bracket, TokenStream::concat(inner).into()
));
r
}
}
Expand All @@ -262,10 +262,9 @@ pub mod rt {

impl ToTokens for () {
fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
vec![TokenTree::Delimited(DelimSpan::dummy(), tokenstream::Delimited {
delim: token::Paren,
tts: TokenStream::empty().into(),
})]
vec![
TokenTree::Delimited(DelimSpan::dummy(), token::Paren, TokenStream::empty().into())
]
}
}

Expand Down Expand Up @@ -382,8 +381,6 @@ pub mod rt {

// Replaces `Token::OpenDelim .. Token::CloseDelim` with `TokenTree::Delimited(..)`.
pub fn unflatten(tts: Vec<TokenTree>) -> Vec<TokenTree> {
use tokenstream::Delimited;

let mut results = Vec::new();
let mut result = Vec::new();
let mut open_span = DUMMY_SP;
Expand All @@ -395,10 +392,11 @@ pub fn unflatten(tts: Vec<TokenTree>) -> Vec<TokenTree> {
}
TokenTree::Token(span, token::CloseDelim(delim)) => {
let delim_span = DelimSpan::from_pair(open_span, span);
let tree = TokenTree::Delimited(delim_span, Delimited {
let tree = TokenTree::Delimited(
delim_span,
delim,
tts: result.into_iter().map(TokenStream::from).collect::<TokenStream>().into(),
});
result.into_iter().map(TokenStream::from).collect::<TokenStream>().into(),
);
result = results.pop().unwrap();
result.push(tree);
}
Expand Down Expand Up @@ -758,10 +756,10 @@ fn statements_mk_tt(cx: &ExtCtxt, tt: &TokenTree, quoted: bool) -> Vec<ast::Stmt
vec![e_tok]);
vec![cx.stmt_expr(e_push)]
},
TokenTree::Delimited(span, ref delimed) => {
let mut stmts = statements_mk_tt(cx, &delimed.open_tt(span.open), false);
stmts.extend(statements_mk_tts(cx, delimed.stream()));
stmts.extend(statements_mk_tt(cx, &delimed.close_tt(span.close), false));
TokenTree::Delimited(span, delim, ref tts) => {
let mut stmts = statements_mk_tt(cx, &TokenTree::open_tt(span.open, delim), false);
stmts.extend(statements_mk_tts(cx, tts.stream()));
stmts.extend(statements_mk_tt(cx, &TokenTree::close_tt(span.close, delim), false));
stmts
}
}
Expand Down
16 changes: 8 additions & 8 deletions src/libsyntax/ext/tt/quoted.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -280,17 +280,17 @@ where
// `tree` is a `$` token. Look at the next token in `trees`
tokenstream::TokenTree::Token(span, token::Dollar) => match trees.next() {
// `tree` is followed by a delimited set of token trees. This indicates the beginning
// of a repetition sequence in the macro (e.g., `$(pat)*`).
Some(tokenstream::TokenTree::Delimited(span, delimited)) => {
// of a repetition sequence in the macro (e.g. `$(pat)*`).
Some(tokenstream::TokenTree::Delimited(span, delim, tts)) => {
// Must have `(` not `{` or `[`
if delimited.delim != token::Paren {
let tok = pprust::token_to_string(&token::OpenDelim(delimited.delim));
if delim != token::Paren {
let tok = pprust::token_to_string(&token::OpenDelim(delim));
let msg = format!("expected `(`, found `{}`", tok);
sess.span_diagnostic.span_err(span.entire(), &msg);
}
// Parse the contents of the sequence itself
let sequence = parse(
delimited.tts.into(),
tts.into(),
expect_matchers,
sess,
features,
Expand Down Expand Up @@ -354,12 +354,12 @@ where

// `tree` is the beginning of a delimited set of tokens (e.g., `(` or `{`). We need to
// descend into the delimited set and further parse it.
tokenstream::TokenTree::Delimited(span, delimited) => TokenTree::Delimited(
tokenstream::TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited(
span,
Lrc::new(Delimited {
delim: delimited.delim,
delim: delim,
tts: parse(
delimited.tts.into(),
tts.into(),
expect_matchers,
sess,
features,
Expand Down
11 changes: 6 additions & 5 deletions src/libsyntax/ext/tt/transcribe.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ use fold::noop_fold_tt;
use parse::token::{self, Token, NtTT};
use smallvec::SmallVec;
use syntax_pos::DUMMY_SP;
use tokenstream::{TokenStream, TokenTree, Delimited, DelimSpan};
use tokenstream::{TokenStream, TokenTree, DelimSpan};

use rustc_data_structures::fx::FxHashMap;
use rustc_data_structures::sync::Lrc;
Expand Down Expand Up @@ -105,10 +105,11 @@ pub fn transcribe(cx: &ExtCtxt,
if result_stack.is_empty() {
return TokenStream::concat(result);
}
let tree = TokenTree::Delimited(span, Delimited {
delim: forest.delim,
tts: TokenStream::concat(result).into(),
});
let tree = TokenTree::Delimited(
span,
forest.delim,
TokenStream::concat(result).into(),
);
result = result_stack.pop().unwrap();
result.push(tree.into());
}
Expand Down
8 changes: 3 additions & 5 deletions src/libsyntax/fold.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -605,12 +605,10 @@ pub fn noop_fold_tt<T: Folder>(tt: TokenTree, fld: &mut T) -> TokenTree {
match tt {
TokenTree::Token(span, tok) =>
TokenTree::Token(fld.new_span(span), fld.fold_token(tok)),
TokenTree::Delimited(span, delimed) => TokenTree::Delimited(
TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited(
DelimSpan::from_pair(fld.new_span(span.open), fld.new_span(span.close)),
Delimited {
tts: fld.fold_tts(delimed.stream()).into(),
delim: delimed.delim,
}
delim,
fld.fold_tts(tts.stream()).into(),
),
}
}
Expand Down
9 changes: 5 additions & 4 deletions src/libsyntax/parse/lexer/tokentrees.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
use print::pprust::token_to_string;
use parse::lexer::StringReader;
use parse::{token, PResult};
use tokenstream::{Delimited, DelimSpan, TokenStream, TokenTree};
use tokenstream::{DelimSpan, TokenStream, TokenTree};

impl<'a> StringReader<'a> {
// Parse a stream of tokens into a list of `TokenTree`s, up to an `Eof`.
Expand Down Expand Up @@ -155,10 +155,11 @@ impl<'a> StringReader<'a> {
_ => {}
}

Ok(TokenTree::Delimited(delim_span, Delimited {
Ok(TokenTree::Delimited(
delim_span,
delim,
tts: tts.into(),
}).into())
tts.into(),
).into())
},
token::CloseDelim(_) => {
// An unexpected closing delimiter (i.e., there is no
Expand Down
61 changes: 29 additions & 32 deletions src/libsyntax/parse/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -764,7 +764,7 @@ mod tests {
use attr::first_attr_value_str_by_name;
use parse;
use print::pprust::item_to_string;
use tokenstream::{self, DelimSpan, TokenTree};
use tokenstream::{DelimSpan, TokenTree};
use util::parser_testing::string_to_stream;
use util::parser_testing::{string_to_expr, string_to_item};
use with_globals;
Expand Down Expand Up @@ -795,42 +795,41 @@ mod tests {
Some(&TokenTree::Token(_, token::Ident(name_macro_rules, false))),
Some(&TokenTree::Token(_, token::Not)),
Some(&TokenTree::Token(_, token::Ident(name_zip, false))),
Some(&TokenTree::Delimited(_, ref macro_delimed)),
Some(&TokenTree::Delimited(_, macro_delim, ref macro_tts)),
)
if name_macro_rules.name == "macro_rules"
&& name_zip.name == "zip" => {
let tts = &macro_delimed.stream().trees().collect::<Vec<_>>();
let tts = &macro_tts.stream().trees().collect::<Vec<_>>();
match (tts.len(), tts.get(0), tts.get(1), tts.get(2)) {
(
3,
Some(&TokenTree::Delimited(_, ref first_delimed)),
Some(&TokenTree::Delimited(_, first_delim, ref first_tts)),
Some(&TokenTree::Token(_, token::FatArrow)),
Some(&TokenTree::Delimited(_, ref second_delimed)),
Some(&TokenTree::Delimited(_, second_delim, ref second_tts)),
)
if macro_delimed.delim == token::Paren => {
let tts = &first_delimed.stream().trees().collect::<Vec<_>>();
if macro_delim == token::Paren => {
let tts = &first_tts.stream().trees().collect::<Vec<_>>();
match (tts.len(), tts.get(0), tts.get(1)) {
(
2,
Some(&TokenTree::Token(_, token::Dollar)),
Some(&TokenTree::Token(_, token::Ident(ident, false))),
)
if first_delimed.delim == token::Paren && ident.name == "a" => {},
_ => panic!("value 3: {:?}", *first_delimed),
if first_delim == token::Paren && ident.name == "a" => {},
_ => panic!("value 3: {:?} {:?}", first_delim, first_tts),
}
let tts = &second_delimed.stream().trees().collect::<Vec<_>>();
let tts = &second_tts.stream().trees().collect::<Vec<_>>();
match (tts.len(), tts.get(0), tts.get(1)) {
(
2,
Some(&TokenTree::Token(_, token::Dollar)),
Some(&TokenTree::Token(_, token::Ident(ident, false))),
)
if second_delimed.delim == token::Paren
&& ident.name == "a" => {},
_ => panic!("value 4: {:?}", *second_delimed),
if second_delim == token::Paren && ident.name == "a" => {},
_ => panic!("value 4: {:?} {:?}", second_delim, second_tts),
}
},
_ => panic!("value 2: {:?}", *macro_delimed),
_ => panic!("value 2: {:?} {:?}", macro_delim, macro_tts),
}
},
_ => panic!("value: {:?}",tts),
Expand All @@ -848,26 +847,24 @@ mod tests {
TokenTree::Token(sp(3, 4), token::Ident(Ident::from_str("a"), false)).into(),
TokenTree::Delimited(
DelimSpan::from_pair(sp(5, 6), sp(13, 14)),
tokenstream::Delimited {
delim: token::DelimToken::Paren,
tts: TokenStream::concat(vec![
TokenTree::Token(sp(6, 7),
token::Ident(Ident::from_str("b"), false)).into(),
TokenTree::Token(sp(8, 9), token::Colon).into(),
TokenTree::Token(sp(10, 13),
token::Ident(Ident::from_str("i32"), false)).into(),
]).into(),
}).into(),
token::DelimToken::Paren,
TokenStream::concat(vec![
TokenTree::Token(sp(6, 7),
token::Ident(Ident::from_str("b"), false)).into(),
TokenTree::Token(sp(8, 9), token::Colon).into(),
TokenTree::Token(sp(10, 13),
token::Ident(Ident::from_str("i32"), false)).into(),
]).into(),
).into(),
TokenTree::Delimited(
DelimSpan::from_pair(sp(15, 16), sp(20, 21)),
tokenstream::Delimited {
delim: token::DelimToken::Brace,
tts: TokenStream::concat(vec![
TokenTree::Token(sp(17, 18),
token::Ident(Ident::from_str("b"), false)).into(),
TokenTree::Token(sp(18, 19), token::Semi).into(),
]).into(),
}).into()
token::DelimToken::Brace,
TokenStream::concat(vec![
TokenTree::Token(sp(17, 18),
token::Ident(Ident::from_str("b"), false)).into(),
TokenTree::Token(sp(18, 19), token::Semi).into(),
]).into(),
).into()
]);

assert_eq!(tts, expected);
Expand Down
Loading