From e498b5d2f93d19909017079f43b7fac937029630 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 14 Oct 2019 10:37:21 +1100 Subject: [PATCH 1/4] Make `TokenStream::from_iter` less general and more efficient. The current code has this impl: ``` impl> iter::FromIterator for TokenStream ``` If given an `IntoIterator`, it will convert each individual `TokenTree` to a `TokenStream` (at the cost of two allocations: a `Vec` and an `Lrc`). It will then merge those `TokenStream`s into a single `TokenStream`. This is inefficient. This commit changes the impl to this less general one: ``` impl iter::FromIterator for TokenStream ``` It collects the `TokenTree`s into a single `Vec` first and then converts that to a `TokenStream` by wrapping it in a single `Lrc`. The previous generality was unnecessary; no other code needs changing. This change speeds up several benchmarks by up to 4%. --- src/libsyntax/tokenstream.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index 0ff1c26bac2b2..0eb35c220729b 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -202,9 +202,9 @@ impl From for TreeAndJoint { } } -impl> iter::FromIterator for TokenStream { - fn from_iter>(iter: I) -> Self { - TokenStream::from_streams(iter.into_iter().map(Into::into).collect::>()) +impl iter::FromIterator for TokenStream { + fn from_iter>(iter: I) -> Self { + TokenStream::new(iter.into_iter().map(Into::into).collect::>()) } } From dbf2523e454eeed7b3bd93c2725ca4facaf60bec Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 14 Oct 2019 11:24:46 +1100 Subject: [PATCH 2/4] Change `Lit::tokens()` to `Lit::token_tree()`. Because most of the call sites have an easier time working with a `TokenTree` instead of a `TokenStream`. --- src/libsyntax/attr/mod.rs | 9 +++++---- src/libsyntax/parse/attr.rs | 9 ++++----- src/libsyntax/parse/literal.rs | 8 ++++---- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs index d291e502c2553..1ce860dc4287e 100644 --- a/src/libsyntax/attr/mod.rs +++ b/src/libsyntax/attr/mod.rs @@ -554,9 +554,10 @@ impl MetaItemKind { match *self { MetaItemKind::Word => TokenStream::default(), MetaItemKind::NameValue(ref lit) => { - let mut vec = vec![TokenTree::token(token::Eq, span).into()]; - lit.tokens().append_to_tree_and_joint_vec(&mut vec); - TokenStream::new(vec) + TokenStream::new(vec![ + TokenTree::token(token::Eq, span).into(), + lit.token_tree().into(), + ]) } MetaItemKind::List(ref list) => { let mut tokens = Vec::new(); @@ -619,7 +620,7 @@ impl NestedMetaItem { fn tokens(&self) -> TokenStream { match *self { NestedMetaItem::MetaItem(ref item) => item.tokens(), - NestedMetaItem::Literal(ref lit) => lit.tokens(), + NestedMetaItem::Literal(ref lit) => lit.token_tree().into(), } } diff --git a/src/libsyntax/parse/attr.rs b/src/libsyntax/parse/attr.rs index 0963efcfc8ac0..bc08684f28062 100644 --- a/src/libsyntax/parse/attr.rs +++ b/src/libsyntax/parse/attr.rs @@ -7,7 +7,6 @@ use crate::tokenstream::{TokenStream, TokenTree}; use crate::source_map::Span; use log::debug; -use smallvec::smallvec; #[derive(Debug)] enum InnerAttributeParsePolicy<'a> { @@ -193,15 +192,15 @@ impl<'a> Parser<'a> { is_interpolated_expr = true; } } - let tokens = if is_interpolated_expr { + let token_tree = if is_interpolated_expr { // We need to accept arbitrary interpolated expressions to continue // supporting things like `doc = $expr` that work on stable. // Non-literal interpolated expressions are rejected after expansion. - self.parse_token_tree().into() + self.parse_token_tree() } else { - self.parse_unsuffixed_lit()?.tokens() + self.parse_unsuffixed_lit()?.token_tree() }; - TokenStream::from_streams(smallvec![eq.into(), tokens]) + TokenStream::new(vec![eq.into(), token_tree.into()]) } else { TokenStream::default() }; diff --git a/src/libsyntax/parse/literal.rs b/src/libsyntax/parse/literal.rs index 14e1696610a4d..7952e293a532d 100644 --- a/src/libsyntax/parse/literal.rs +++ b/src/libsyntax/parse/literal.rs @@ -3,7 +3,7 @@ use crate::ast::{self, Lit, LitKind}; use crate::parse::token::{self, Token}; use crate::symbol::{kw, sym, Symbol}; -use crate::tokenstream::{TokenStream, TokenTree}; +use crate::tokenstream::TokenTree; use log::debug; use rustc_data_structures::sync::Lrc; @@ -216,13 +216,13 @@ impl Lit { Lit { token: kind.to_lit_token(), kind, span } } - /// Losslessly convert an AST literal into a token stream. - crate fn tokens(&self) -> TokenStream { + /// Losslessly convert an AST literal into a token tree. + crate fn token_tree(&self) -> TokenTree { let token = match self.token.kind { token::Bool => token::Ident(self.token.symbol, false), _ => token::Literal(self.token), }; - TokenTree::token(token, self.span).into() + TokenTree::token(token, self.span) } } From de0188904b3e13f39550044e88cf3131037cd6f7 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 14 Oct 2019 14:06:00 +1100 Subject: [PATCH 3/4] Change `MetaItem::tokens()` to `MetaItem::token_trees_and_joints()`. Likewise for `MetaItemKind::tokens()` and `NestedMetaItem::tokens()`. This avoids some unnecessary `TokenTree` to `TokenStream` conversions, and removes the need for the clumsy `TokenStream::append_to_tree_and_joint_vec()`. --- src/libsyntax/attr/mod.rs | 57 +++++++++++++++++++++++------------- src/libsyntax/ext/expand.rs | 5 +++- src/libsyntax/tokenstream.rs | 4 --- 3 files changed, 40 insertions(+), 26 deletions(-) diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs index 1ce860dc4287e..2733b1b91c915 100644 --- a/src/libsyntax/attr/mod.rs +++ b/src/libsyntax/attr/mod.rs @@ -22,7 +22,7 @@ use crate::ptr::P; use crate::sess::ParseSess; use crate::symbol::{sym, Symbol}; use crate::ThinVec; -use crate::tokenstream::{TokenStream, TokenTree, DelimSpan}; +use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint}; use crate::GLOBALS; use log::debug; @@ -340,7 +340,10 @@ impl Attribute { DUMMY_SP, ); f(&Attribute { - item: AttrItem { path: meta.path, tokens: meta.kind.tokens(meta.span) }, + item: AttrItem { + path: meta.path, + tokens: TokenStream::new(meta.kind.token_trees_and_joints(meta.span)), + }, id: self.id, style: self.style, is_sugared_doc: true, @@ -400,12 +403,22 @@ pub fn mk_attr(style: AttrStyle, path: Path, tokens: TokenStream, span: Span) -> /// Returns an inner attribute with the given value and span. pub fn mk_attr_inner(item: MetaItem) -> Attribute { - mk_attr(AttrStyle::Inner, item.path, item.kind.tokens(item.span), item.span) + mk_attr( + AttrStyle::Inner, + item.path, + TokenStream::new(item.kind.token_trees_and_joints(item.span)), + item.span, + ) } /// Returns an outer attribute with the given value and span. pub fn mk_attr_outer(item: MetaItem) -> Attribute { - mk_attr(AttrStyle::Outer, item.path, item.kind.tokens(item.span), item.span) + mk_attr( + AttrStyle::Outer, + item.path, + TokenStream::new(item.kind.token_trees_and_joints(item.span)), + item.span, + ) } pub fn mk_sugared_doc_attr(text: Symbol, span: Span) -> Attribute { @@ -415,7 +428,7 @@ pub fn mk_sugared_doc_attr(text: Symbol, span: Span) -> Attribute { Attribute { item: AttrItem { path: Path::from_ident(Ident::with_dummy_span(sym::doc).with_span_pos(span)), - tokens: MetaItemKind::NameValue(lit).tokens(span), + tokens: TokenStream::new(MetaItemKind::NameValue(lit).token_trees_and_joints(span)), }, id: mk_attr_id(), style, @@ -476,7 +489,7 @@ pub fn first_attr_value_str_by_name(attrs: &[Attribute], name: Symbol) -> Option } impl MetaItem { - fn tokens(&self) -> TokenStream { + fn token_trees_and_joints(&self) -> Vec { let mut idents = vec![]; let mut last_pos = BytePos(0 as u32); for (i, segment) in self.path.segments.iter().enumerate() { @@ -490,8 +503,8 @@ impl MetaItem { idents.push(TokenTree::Token(Token::from_ast_ident(segment.ident)).into()); last_pos = segment.ident.span.hi(); } - self.kind.tokens(self.span).append_to_tree_and_joint_vec(&mut idents); - TokenStream::new(idents) + idents.extend(self.kind.token_trees_and_joints(self.span)); + idents } fn from_tokens(tokens: &mut iter::Peekable) -> Option @@ -550,14 +563,14 @@ impl MetaItem { } impl MetaItemKind { - pub fn tokens(&self, span: Span) -> TokenStream { + pub fn token_trees_and_joints(&self, span: Span) -> Vec { match *self { - MetaItemKind::Word => TokenStream::default(), + MetaItemKind::Word => vec![], MetaItemKind::NameValue(ref lit) => { - TokenStream::new(vec![ + vec![ TokenTree::token(token::Eq, span).into(), lit.token_tree().into(), - ]) + ] } MetaItemKind::List(ref list) => { let mut tokens = Vec::new(); @@ -565,13 +578,15 @@ impl MetaItemKind { if i > 0 { tokens.push(TokenTree::token(token::Comma, span).into()); } - item.tokens().append_to_tree_and_joint_vec(&mut tokens); + tokens.extend(item.token_trees_and_joints()) } - TokenTree::Delimited( - DelimSpan::from_single(span), - token::Paren, - TokenStream::new(tokens).into(), - ).into() + vec![ + TokenTree::Delimited( + DelimSpan::from_single(span), + token::Paren, + TokenStream::new(tokens).into(), + ).into() + ] } } } @@ -617,10 +632,10 @@ impl NestedMetaItem { } } - fn tokens(&self) -> TokenStream { + fn token_trees_and_joints(&self) -> Vec { match *self { - NestedMetaItem::MetaItem(ref item) => item.tokens(), - NestedMetaItem::Literal(ref lit) => lit.token_tree().into(), + NestedMetaItem::MetaItem(ref item) => item.token_trees_and_joints(), + NestedMetaItem::Literal(ref lit) => vec![lit.token_tree().into()], } } diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs index 2559e87476277..c22f84388d196 100644 --- a/src/libsyntax/ext/expand.rs +++ b/src/libsyntax/ext/expand.rs @@ -1487,7 +1487,10 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> { let meta = attr::mk_list_item(Ident::with_dummy_span(sym::doc), items); *at = attr::Attribute { - item: AttrItem { path: meta.path, tokens: meta.kind.tokens(meta.span) }, + item: AttrItem { + path: meta.path, + tokens: TokenStream::new(meta.kind.token_trees_and_joints(meta.span)), + }, span: at.span, id: at.id, style: at.style, diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index 0eb35c220729b..d7e26dbc2cca8 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -271,10 +271,6 @@ impl TokenStream { } } - pub fn append_to_tree_and_joint_vec(self, vec: &mut Vec) { - vec.extend(self.0.iter().cloned()); - } - pub fn trees(&self) -> Cursor { self.clone().into_trees() } From 08afe1ffce7cba9b55c112d31535b80904f2201d Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 16 Oct 2019 09:30:28 +1100 Subject: [PATCH 4/4] Remove two no-op `into()` calls. --- src/libsyntax/parse/parser.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 86383761484db..34068dae8deda 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -285,10 +285,10 @@ impl TokenCursor { token::NoDelim, &if doc_comment_style(&name.as_str()) == AttrStyle::Inner { [TokenTree::token(token::Pound, sp), TokenTree::token(token::Not, sp), body] - .iter().cloned().collect::().into() + .iter().cloned().collect::() } else { [TokenTree::token(token::Pound, sp), body] - .iter().cloned().collect::().into() + .iter().cloned().collect::() }, )));