From 08857c1c5743133ac309820640ef0755c82fedb7 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 12 Dec 2018 08:46:18 +1100 Subject: [PATCH 1/6] XXX: SymbolIndex --- src/libsyntax/parse/token.rs | 4 +++ src/libsyntax_pos/lib.rs | 3 ++ src/libsyntax_pos/symbol.rs | 61 +++++++++++++++++++++++------------- 3 files changed, 46 insertions(+), 22 deletions(-) diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 8e4d3c0166bb2..ed7466574596f 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -207,6 +207,10 @@ pub enum Token { Eof, } +// `Token` is used a lot. Make sure it doesn't unintentionally get bigger. +#[cfg(target_arch = "x86_64")] +static_assert!(MEM_SIZE_OF_STATEMENT: mem::size_of::() == 16); + impl Token { pub fn interpolated(nt: Nonterminal) -> Token { Token::Interpolated(Lrc::new((nt, LazyTokenStream::new()))) diff --git a/src/libsyntax_pos/lib.rs b/src/libsyntax_pos/lib.rs index 8b7ffa499cd71..9aafb9fc549fc 100644 --- a/src/libsyntax_pos/lib.rs +++ b/src/libsyntax_pos/lib.rs @@ -24,10 +24,13 @@ #![feature(nll)] #![feature(non_exhaustive)] #![feature(optin_builtin_traits)] +#![feature(rustc_attrs)] #![feature(specialization)] +#![feature(step_trait)] #![cfg_attr(not(stage0), feature(stdsimd))] extern crate arena; +#[macro_use] extern crate rustc_data_structures; #[macro_use] diff --git a/src/libsyntax_pos/symbol.rs b/src/libsyntax_pos/symbol.rs index 05c53878e7004..3d87134511e72 100644 --- a/src/libsyntax_pos/symbol.rs +++ b/src/libsyntax_pos/symbol.rs @@ -14,6 +14,7 @@ use arena::DroplessArena; use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::indexed_vec::Idx; use serialize::{Decodable, Decoder, Encodable, Encoder}; use std::fmt; @@ -143,9 +144,18 @@ impl Decodable for Ident { } } -/// A symbol is an interned or gensymed string. +/// A symbol is an interned or gensymed string. The use of newtype_index! means +/// that Option only takes up 4 bytes, because newtype_index! reserves +/// the last 256 values for tagging purposes. +/// +/// Note that Symbol cannot be a newtype_index! directly because it implements +/// fmt::Debug, Encodable, and Decodable in special ways. #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct Symbol(u32); +pub struct Symbol(SymbolIndex); + +newtype_index! { + pub struct SymbolIndex { .. } +} // The interner is pointed to by a thread local value which is only set on the main thread // with parallelization is disabled. So we don't allow `Symbol` to transfer between threads @@ -156,6 +166,10 @@ impl !Send for Symbol { } impl !Sync for Symbol { } impl Symbol { + const fn new(n: u32) -> Self { + Symbol(SymbolIndex::from_u32_const(n)) + } + /// Maps a string to its interned representation. pub fn intern(string: &str) -> Self { with_interner(|interner| interner.intern(string)) @@ -189,7 +203,7 @@ impl Symbol { } pub fn as_u32(self) -> u32 { - self.0 + self.0.as_u32() } } @@ -197,7 +211,7 @@ impl fmt::Debug for Symbol { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let is_gensymed = with_interner(|interner| interner.is_gensymed(*self)); if is_gensymed { - write!(f, "{}({})", self, self.0) + write!(f, "{}({:?})", self, self.0) } else { write!(f, "{}", self) } @@ -229,6 +243,9 @@ impl> PartialEq for Symbol { } // The `&'static str`s in this type actually point into the arena. +// +// Note that normal symbols are indexed upward from 0, and gensyms are indexed +// downward from SymbolIndex::MAX_AS_U32. #[derive(Default)] pub struct Interner { arena: DroplessArena, @@ -243,7 +260,7 @@ impl Interner { for &string in init { if string == "" { // We can't allocate empty strings in the arena, so handle this here. - let name = Symbol(this.strings.len() as u32); + let name = Symbol::new(this.strings.len() as u32); this.names.insert("", name); this.strings.push(""); } else { @@ -258,7 +275,7 @@ impl Interner { return name; } - let name = Symbol(self.strings.len() as u32); + let name = Symbol::new(self.strings.len() as u32); // `from_utf8_unchecked` is safe since we just allocated a `&str` which is known to be // UTF-8. @@ -276,10 +293,10 @@ impl Interner { } pub fn interned(&self, symbol: Symbol) -> Symbol { - if (symbol.0 as usize) < self.strings.len() { + if (symbol.0.as_usize()) < self.strings.len() { symbol } else { - self.interned(self.gensyms[(!0 - symbol.0) as usize]) + self.interned(self.gensyms[(SymbolIndex::MAX_AS_U32 - symbol.0.as_u32()) as usize]) } } @@ -290,17 +307,17 @@ impl Interner { fn gensymed(&mut self, symbol: Symbol) -> Symbol { self.gensyms.push(symbol); - Symbol(!0 - self.gensyms.len() as u32 + 1) + Symbol::new(SymbolIndex::MAX_AS_U32 - self.gensyms.len() as u32 + 1) } fn is_gensymed(&mut self, symbol: Symbol) -> bool { - symbol.0 as usize >= self.strings.len() + symbol.0.as_usize() >= self.strings.len() } pub fn get(&self, symbol: Symbol) -> &str { - match self.strings.get(symbol.0 as usize) { + match self.strings.get(symbol.0.as_usize()) { Some(string) => string, - None => self.get(self.gensyms[(!0 - symbol.0) as usize]), + None => self.get(self.gensyms[(SymbolIndex::MAX_AS_U32 - symbol.0.as_u32()) as usize]), } } } @@ -324,7 +341,7 @@ macro_rules! declare_keywords {( $( #[allow(non_upper_case_globals)] pub const $konst: Keyword = Keyword { - ident: Ident::with_empty_ctxt(super::Symbol($index)) + ident: Ident::with_empty_ctxt(super::Symbol::new($index)) }; )* @@ -709,19 +726,19 @@ mod tests { fn interner_tests() { let mut i: Interner = Interner::default(); // first one is zero: - assert_eq!(i.intern("dog"), Symbol(0)); + assert_eq!(i.intern("dog"), Symbol::new(0)); // re-use gets the same entry: - assert_eq!(i.intern("dog"), Symbol(0)); + assert_eq!(i.intern("dog"), Symbol::new(0)); // different string gets a different #: - assert_eq!(i.intern("cat"), Symbol(1)); - assert_eq!(i.intern("cat"), Symbol(1)); + assert_eq!(i.intern("cat"), Symbol::new(1)); + assert_eq!(i.intern("cat"), Symbol::new(1)); // dog is still at zero - assert_eq!(i.intern("dog"), Symbol(0)); - assert_eq!(i.gensym("zebra"), Symbol(4294967295)); - // gensym of same string gets new number : - assert_eq!(i.gensym("zebra"), Symbol(4294967294)); + assert_eq!(i.intern("dog"), Symbol::new(0)); + assert_eq!(i.gensym("zebra"), Symbol::new(SymbolIndex::MAX_AS_U32)); + // gensym of same string gets new number: + assert_eq!(i.gensym("zebra"), Symbol::new(SymbolIndex::MAX_AS_U32 - 1)); // gensym of *existing* string gets new number: - assert_eq!(i.gensym("dog"), Symbol(4294967293)); + assert_eq!(i.gensym("dog"), Symbol::new(SymbolIndex::MAX_AS_U32 - 2)); } #[test] From 67f8fb5960551fbe80cc49ace1103a9fe393f9f4 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 12 Dec 2018 09:01:39 +1100 Subject: [PATCH 2/6] Use `Lrc>` instead of `RcVec`. This shrinks: - ThinTokenStream: 16 to 8 bytes - TokenTree: 32 to 24 bytes - TokenStream: 40 to 32 bytes The only downside is that in a couple of places this requires using `to_vec()` (which allocates) instead of `sub_slice()`. But those places are rarely executed, so it doesn't hurt perf. Overall, this reduces instruction counts on numerous benchmarks by up to 3%. --- src/libsyntax/tokenstream.rs | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index 90191c54126d1..da7f5029d9db6 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -28,8 +28,8 @@ use ext::tt::{macro_parser, quoted}; use parse::Directory; use parse::token::{self, DelimToken, Token}; use print::pprust; +use rustc_data_structures::sync::Lrc; use serialize::{Decoder, Decodable, Encoder, Encodable}; -use util::RcVec; use std::borrow::Cow; use std::{fmt, iter, mem}; @@ -160,7 +160,7 @@ pub struct TokenStream { // `TokenStream` is used a lot. Make sure it doesn't unintentionally get bigger. #[cfg(target_arch = "x86_64")] -static_assert!(MEM_SIZE_OF_TOKEN_STREAM: mem::size_of::() == 40); +static_assert!(MEM_SIZE_OF_TOKEN_STREAM: mem::size_of::() == 32); impl TokenStream { /// Given a `TokenStream` with a `Stream` of only two arguments, return a new `TokenStream` @@ -194,7 +194,7 @@ impl TokenStream { new_slice.extend_from_slice(parts.0); new_slice.push(comma); new_slice.extend_from_slice(parts.1); - let slice = RcVec::new(new_slice); + let slice = Lrc::new(new_slice); return Some((TokenStream { kind: TokenStreamKind::Stream(slice) }, sp)); } } @@ -207,7 +207,7 @@ enum TokenStreamKind { Empty, Tree(TokenTree), JointTree(TokenTree), - Stream(RcVec), + Stream(Lrc>), } impl From for TokenStream { @@ -246,7 +246,7 @@ impl Extend for TokenStream { vec.push(TokenStream { kind }); vec } - TokenStreamKind::Stream(rc_vec) => match RcVec::try_unwrap(rc_vec) { + TokenStreamKind::Stream(rc_vec) => match Lrc::try_unwrap(rc_vec) { Ok(mut vec) => { // Extend in place using the existing capacity if possible. // This is the fast path for libraries like `quote` that @@ -277,7 +277,7 @@ impl Extend for TokenStream { *self = match tts.len() { 0 => TokenStream::empty(), 1 => tts.pop().unwrap(), - _ => TokenStream::concat_rc_vec(RcVec::new_preserving_capacity(tts)), + _ => TokenStream::concat_rc_vec(Lrc::new(tts)), }; } } @@ -314,11 +314,11 @@ impl TokenStream { match streams.len() { 0 => TokenStream::empty(), 1 => streams.pop().unwrap(), - _ => TokenStream::concat_rc_vec(RcVec::new(streams)), + _ => TokenStream::concat_rc_vec(Lrc::new(streams)), } } - fn concat_rc_vec(streams: RcVec) -> TokenStream { + fn concat_rc_vec(streams: Lrc>) -> TokenStream { TokenStream { kind: TokenStreamKind::Stream(streams) } } @@ -483,7 +483,7 @@ impl TokenStreamBuilder { match len { 1 => {} 2 => self.0.push(streams[0].clone().into()), - _ => self.0.push(TokenStream::concat_rc_vec(streams.sub_slice(0 .. len - 1))), + _ => self.0.push(TokenStream::concat_rc_vec( Lrc::new(streams[0 .. len - 1].to_vec()))), } self.push_all_but_last_tree(&streams[len - 1]) } @@ -495,7 +495,7 @@ impl TokenStreamBuilder { match len { 1 => {} 2 => self.0.push(streams[1].clone().into()), - _ => self.0.push(TokenStream::concat_rc_vec(streams.sub_slice(1 .. len))), + _ => self.0.push(TokenStream::concat_rc_vec(Lrc::new(streams[1 .. len].to_vec()))), } self.push_all_but_first_tree(&streams[0]) } @@ -515,13 +515,13 @@ enum CursorKind { #[derive(Clone)] struct StreamCursor { - stream: RcVec, + stream: Lrc>, index: usize, - stack: Vec<(RcVec, usize)>, + stack: Vec<(Lrc>, usize)>, } impl StreamCursor { - fn new(stream: RcVec) -> Self { + fn new(stream: Lrc>) -> Self { StreamCursor { stream: stream, index: 0, stack: Vec::new() } } @@ -544,7 +544,7 @@ impl StreamCursor { } } - fn insert(&mut self, stream: RcVec) { + fn insert(&mut self, stream: Lrc>) { self.stack.push((mem::replace(&mut self.stream, stream), mem::replace(&mut self.index, 0))); } @@ -656,7 +656,7 @@ impl Cursor { /// `ThinTokenStream` is smaller, but needs to allocate to represent a single `TokenTree`. /// We must use `ThinTokenStream` in `TokenTree::Delimited` to avoid infinite size due to recursion. #[derive(Debug, Clone)] -pub struct ThinTokenStream(Option>); +pub struct ThinTokenStream(Option>>); impl ThinTokenStream { pub fn stream(&self) -> TokenStream { @@ -668,8 +668,8 @@ impl From for ThinTokenStream { fn from(stream: TokenStream) -> ThinTokenStream { ThinTokenStream(match stream.kind { TokenStreamKind::Empty => None, - TokenStreamKind::Tree(tree) => Some(RcVec::new(vec![tree.into()])), - TokenStreamKind::JointTree(tree) => Some(RcVec::new(vec![tree.joint()])), + TokenStreamKind::Tree(tree) => Some(Lrc::new(vec![tree.into()])), + TokenStreamKind::JointTree(tree) => Some(Lrc::new(vec![tree.joint()])), TokenStreamKind::Stream(stream) => Some(stream), }) } From 3c9aef1f452d9b7063d908fc40012e39ae1d2d16 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 12 Dec 2018 09:40:22 +1100 Subject: [PATCH 3/6] Use `TokenStream::concat` more. It's a better choice in a few places. --- src/libsyntax/tokenstream.rs | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index da7f5029d9db6..e433f8d8dfdb4 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -167,9 +167,9 @@ impl TokenStream { /// separating the two arguments with a comma for diagnostic suggestions. pub(crate) fn add_comma(&self) -> Option<(TokenStream, Span)> { // Used to suggest if a user writes `foo!(a b);` - if let TokenStreamKind::Stream(ref slice) = self.kind { + if let TokenStreamKind::Stream(ref stream) = self.kind { let mut suggestion = None; - let mut iter = slice.iter().enumerate().peekable(); + let mut iter = stream.iter().enumerate().peekable(); while let Some((pos, ts)) = iter.next() { if let Some((_, next)) = iter.peek() { let sp = match (&ts.kind, &next.kind) { @@ -189,13 +189,12 @@ impl TokenStream { } } if let Some((pos, comma, sp)) = suggestion { - let mut new_slice = vec![]; - let parts = slice.split_at(pos + 1); - new_slice.extend_from_slice(parts.0); - new_slice.push(comma); - new_slice.extend_from_slice(parts.1); - let slice = Lrc::new(new_slice); - return Some((TokenStream { kind: TokenStreamKind::Stream(slice) }, sp)); + let mut new_stream = vec![]; + let parts = stream.split_at(pos + 1); + new_stream.extend_from_slice(parts.0); + new_stream.push(comma); + new_stream.extend_from_slice(parts.1); + return Some((TokenStream::concat(new_stream), sp)); } } None @@ -273,12 +272,7 @@ impl Extend for TokenStream { // Build the resulting token stream. If it contains more than one token, // preserve capacity in the vector in anticipation of the caller // performing additional calls to extend. - let mut tts = builder.0; - *self = match tts.len() { - 0 => TokenStream::empty(), - 1 => tts.pop().unwrap(), - _ => TokenStream::concat_rc_vec(Lrc::new(tts)), - }; + *self = TokenStream::concat(builder.0); } } @@ -483,7 +477,7 @@ impl TokenStreamBuilder { match len { 1 => {} 2 => self.0.push(streams[0].clone().into()), - _ => self.0.push(TokenStream::concat_rc_vec( Lrc::new(streams[0 .. len - 1].to_vec()))), + _ => self.0.push(TokenStream::concat(streams[0 .. len - 1].to_vec())), } self.push_all_but_last_tree(&streams[len - 1]) } @@ -495,7 +489,7 @@ impl TokenStreamBuilder { match len { 1 => {} 2 => self.0.push(streams[1].clone().into()), - _ => self.0.push(TokenStream::concat_rc_vec(Lrc::new(streams[1 .. len].to_vec()))), + _ => self.0.push(TokenStream::concat(streams[1 .. len].to_vec())), } self.push_all_but_first_tree(&streams[0]) } From 43c6eced848034cf10018b109245263e2428bb1f Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 12 Dec 2018 09:42:54 +1100 Subject: [PATCH 4/6] Remove `RcVec` and `RcSlice`. They're both unused now. --- src/libsyntax/lib.rs | 6 --- src/libsyntax/util/rc_slice.rs | 64 ------------------------ src/libsyntax/util/rc_vec.rs | 90 ---------------------------------- src/tools/linkchecker/main.rs | 1 - 4 files changed, 161 deletions(-) delete mode 100644 src/libsyntax/util/rc_slice.rs delete mode 100644 src/libsyntax/util/rc_vec.rs diff --git a/src/libsyntax/lib.rs b/src/libsyntax/lib.rs index 0c24c20554acc..1fa11a4d6c856 100644 --- a/src/libsyntax/lib.rs +++ b/src/libsyntax/lib.rs @@ -145,12 +145,6 @@ pub mod util { #[cfg(test)] pub mod parser_testing; pub mod move_map; - - mod rc_slice; - pub use self::rc_slice::RcSlice; - - mod rc_vec; - pub use self::rc_vec::RcVec; } pub mod json; diff --git a/src/libsyntax/util/rc_slice.rs b/src/libsyntax/util/rc_slice.rs deleted file mode 100644 index 520b7a48e3025..0000000000000 --- a/src/libsyntax/util/rc_slice.rs +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2017 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -use std::fmt; -use std::ops::{Deref, Range}; -use rustc_data_structures::sync::Lrc; - -use rustc_data_structures::stable_hasher::{StableHasher, StableHasherResult, - HashStable}; - -#[derive(Clone)] -pub struct RcSlice { - data: Lrc>, - offset: u32, - len: u32, -} - -impl RcSlice { - pub fn new(vec: Vec) -> Self { - RcSlice { - offset: 0, - len: vec.len() as u32, - data: Lrc::new(vec.into_boxed_slice()), - } - } - - pub fn sub_slice(&self, range: Range) -> Self { - RcSlice { - data: self.data.clone(), - offset: self.offset + range.start as u32, - len: (range.end - range.start) as u32, - } - } -} - -impl Deref for RcSlice { - type Target = [T]; - fn deref(&self) -> &[T] { - &self.data[self.offset as usize .. (self.offset + self.len) as usize] - } -} - -impl fmt::Debug for RcSlice { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - fmt::Debug::fmt(self.deref(), f) - } -} - -impl HashStable for RcSlice - where T: HashStable -{ - fn hash_stable(&self, - hcx: &mut CTX, - hasher: &mut StableHasher) { - (**self).hash_stable(hcx, hasher); - } -} diff --git a/src/libsyntax/util/rc_vec.rs b/src/libsyntax/util/rc_vec.rs deleted file mode 100644 index 99fbce1ad91e1..0000000000000 --- a/src/libsyntax/util/rc_vec.rs +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2017 The Rust Project Developers. See the COPYRIGHT -// file at the top-level directory of this distribution and at -// http://rust-lang.org/COPYRIGHT. -// -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -use std::fmt; -use std::ops::{Deref, Range}; - -use rustc_data_structures::stable_hasher::{HashStable, StableHasher, StableHasherResult}; -use rustc_data_structures::sync::Lrc; - -#[derive(Clone)] -pub struct RcVec { - data: Lrc>, - offset: u32, - len: u32, -} - -impl RcVec { - pub fn new(mut vec: Vec) -> Self { - // By default, constructing RcVec from Vec gives it just enough capacity - // to hold the initial elements. Callers that anticipate needing to - // extend the vector may prefer RcVec::new_preserving_capacity. - vec.shrink_to_fit(); - Self::new_preserving_capacity(vec) - } - - pub fn new_preserving_capacity(vec: Vec) -> Self { - RcVec { - offset: 0, - len: vec.len() as u32, - data: Lrc::new(vec), - } - } - - pub fn sub_slice(&self, range: Range) -> Self { - RcVec { - data: self.data.clone(), - offset: self.offset + range.start as u32, - len: (range.end - range.start) as u32, - } - } - - /// If this RcVec has exactly one strong reference, returns ownership of the - /// underlying vector. Otherwise returns self unmodified. - pub fn try_unwrap(self) -> Result, Self> { - match Lrc::try_unwrap(self.data) { - // If no other RcVec shares ownership of this data. - Ok(mut vec) => { - // Drop any elements after our view of the data. - vec.truncate(self.offset as usize + self.len as usize); - // Drop any elements before our view of the data. Do this after - // the `truncate` so that elements past the end of our view do - // not need to be copied around. - vec.drain(..self.offset as usize); - Ok(vec) - } - - // If the data is shared. - Err(data) => Err(RcVec { data, ..self }), - } - } -} - -impl Deref for RcVec { - type Target = [T]; - fn deref(&self) -> &[T] { - &self.data[self.offset as usize..(self.offset + self.len) as usize] - } -} - -impl fmt::Debug for RcVec { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - fmt::Debug::fmt(self.deref(), f) - } -} - -impl HashStable for RcVec -where - T: HashStable, -{ - fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) { - (**self).hash_stable(hcx, hasher); - } -} diff --git a/src/tools/linkchecker/main.rs b/src/tools/linkchecker/main.rs index e6bf9a285723b..1aa647a6a1b78 100644 --- a/src/tools/linkchecker/main.rs +++ b/src/tools/linkchecker/main.rs @@ -137,7 +137,6 @@ fn check(cache: &mut Cache, file.ends_with("symbol/struct.InternedString.html") || file.ends_with("ast/struct.ThinVec.html") || file.ends_with("util/struct.ThinVec.html") || - file.ends_with("util/struct.RcSlice.html") || file.ends_with("layout/struct.TyLayout.html") || file.ends_with("humantime/struct.Timestamp.html") || file.ends_with("log/index.html") || From 07c12fa89e297bf5940f903dea8186a0a33f9d82 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 12 Dec 2018 09:55:43 +1100 Subject: [PATCH 5/6] Merge `TokenStreamKind` into `TokenStream`. Because the distinction provides little value, and removing it cleans up the code quite a bit. --- src/libsyntax/tokenstream.rs | 137 +++++++++++++++++------------------ 1 file changed, 65 insertions(+), 72 deletions(-) diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index e433f8d8dfdb4..e31bde4082dad 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -123,7 +123,7 @@ impl TokenTree { } pub fn joint(self) -> TokenStream { - TokenStream { kind: TokenStreamKind::JointTree(self) } + TokenStream::JointTree(self) } /// Returns the opening delimiter as a token tree. @@ -154,8 +154,11 @@ impl TokenTree { /// instead of a representation of the abstract syntax tree. /// Today's `TokenTree`s can still contain AST via `Token::Interpolated` for back-compat. #[derive(Clone, Debug)] -pub struct TokenStream { - kind: TokenStreamKind, +pub enum TokenStream { + Empty, + Tree(TokenTree), + JointTree(TokenTree), + Stream(Lrc>), } // `TokenStream` is used a lot. Make sure it doesn't unintentionally get bigger. @@ -167,24 +170,22 @@ impl TokenStream { /// separating the two arguments with a comma for diagnostic suggestions. pub(crate) fn add_comma(&self) -> Option<(TokenStream, Span)> { // Used to suggest if a user writes `foo!(a b);` - if let TokenStreamKind::Stream(ref stream) = self.kind { + if let TokenStream::Stream(ref stream) = self { let mut suggestion = None; let mut iter = stream.iter().enumerate().peekable(); while let Some((pos, ts)) = iter.next() { if let Some((_, next)) = iter.peek() { - let sp = match (&ts.kind, &next.kind) { - (TokenStreamKind::Tree(TokenTree::Token(_, token::Token::Comma)), _) | - (_, TokenStreamKind::Tree(TokenTree::Token(_, token::Token::Comma))) => { + let sp = match (&ts, &next) { + (TokenStream::Tree(TokenTree::Token(_, token::Token::Comma)), _) | + (_, TokenStream::Tree(TokenTree::Token(_, token::Token::Comma))) => { continue; } - (TokenStreamKind::Tree(TokenTree::Token(sp, _)), _) => *sp, - (TokenStreamKind::Tree(TokenTree::Delimited(sp, ..)), _) => sp.entire(), + (TokenStream::Tree(TokenTree::Token(sp, _)), _) => *sp, + (TokenStream::Tree(TokenTree::Delimited(sp, ..)), _) => sp.entire(), _ => continue, }; let sp = sp.shrink_to_hi(); - let comma = TokenStream { - kind: TokenStreamKind::Tree(TokenTree::Token(sp, token::Comma)), - }; + let comma = TokenStream::Tree(TokenTree::Token(sp, token::Comma)); suggestion = Some((pos, comma, sp)); } } @@ -201,17 +202,9 @@ impl TokenStream { } } -#[derive(Clone, Debug)] -enum TokenStreamKind { - Empty, - Tree(TokenTree), - JointTree(TokenTree), - Stream(Lrc>), -} - impl From for TokenStream { fn from(tt: TokenTree) -> TokenStream { - TokenStream { kind: TokenStreamKind::Tree(tt) } + TokenStream::Tree(tt) } } @@ -230,22 +223,22 @@ impl> iter::FromIterator for TokenStream { impl Extend for TokenStream { fn extend>(&mut self, iter: I) { let iter = iter.into_iter(); - let kind = mem::replace(&mut self.kind, TokenStreamKind::Empty); + let this = mem::replace(self, TokenStream::Empty); // Vector of token streams originally in self. - let tts: Vec = match kind { - TokenStreamKind::Empty => { + let tts: Vec = match this { + TokenStream::Empty => { let mut vec = Vec::new(); vec.reserve(iter.size_hint().0); vec } - TokenStreamKind::Tree(_) | TokenStreamKind::JointTree(_) => { + TokenStream::Tree(_) | TokenStream::JointTree(_) => { let mut vec = Vec::new(); vec.reserve(1 + iter.size_hint().0); - vec.push(TokenStream { kind }); + vec.push(this); vec } - TokenStreamKind::Stream(rc_vec) => match Lrc::try_unwrap(rc_vec) { + TokenStream::Stream(rc_vec) => match Lrc::try_unwrap(rc_vec) { Ok(mut vec) => { // Extend in place using the existing capacity if possible. // This is the fast path for libraries like `quote` that @@ -286,7 +279,7 @@ impl PartialEq for TokenStream { impl TokenStream { pub fn len(&self) -> usize { - if let TokenStreamKind::Stream(ref slice) = self.kind { + if let TokenStream::Stream(ref slice) = self { slice.len() } else { 0 @@ -294,12 +287,12 @@ impl TokenStream { } pub fn empty() -> TokenStream { - TokenStream { kind: TokenStreamKind::Empty } + TokenStream::Empty } pub fn is_empty(&self) -> bool { - match self.kind { - TokenStreamKind::Empty => true, + match self { + TokenStream::Empty => true, _ => false, } } @@ -313,7 +306,7 @@ impl TokenStream { } fn concat_rc_vec(streams: Lrc>) -> TokenStream { - TokenStream { kind: TokenStreamKind::Stream(streams) } + TokenStream::Stream(streams) } pub fn trees(&self) -> Cursor { @@ -377,9 +370,9 @@ impl TokenStream { /// Precondition: `self` consists of a single token tree. /// Returns true if the token tree is a joint operation w.r.t. `proc_macro::TokenNode`. pub fn as_tree(self) -> (TokenTree, bool /* joint? */) { - match self.kind { - TokenStreamKind::Tree(tree) => (tree, false), - TokenStreamKind::JointTree(tree) => (tree, true), + match self { + TokenStream::Tree(tree) => (tree, false), + TokenStream::JointTree(tree) => (tree, true), _ => unreachable!(), } } @@ -389,9 +382,9 @@ impl TokenStream { let mut result = Vec::new(); let mut i = 0; while let Some(stream) = trees.next_as_stream() { - result.push(match stream.kind { - TokenStreamKind::Tree(tree) => f(i, tree).into(), - TokenStreamKind::JointTree(tree) => f(i, tree).joint(), + result.push(match stream { + TokenStream::Tree(tree) => f(i, tree).into(), + TokenStream::JointTree(tree) => f(i, tree).joint(), _ => unreachable!() }); i += 1; @@ -403,9 +396,9 @@ impl TokenStream { let mut trees = self.into_trees(); let mut result = Vec::new(); while let Some(stream) = trees.next_as_stream() { - result.push(match stream.kind { - TokenStreamKind::Tree(tree) => f(tree).into(), - TokenStreamKind::JointTree(tree) => f(tree).joint(), + result.push(match stream { + TokenStream::Tree(tree) => f(tree).into(), + TokenStream::JointTree(tree) => f(tree).joint(), _ => unreachable!() }); } @@ -413,19 +406,19 @@ impl TokenStream { } fn first_tree_and_joint(&self) -> Option<(TokenTree, bool)> { - match self.kind { - TokenStreamKind::Empty => None, - TokenStreamKind::Tree(ref tree) => Some((tree.clone(), false)), - TokenStreamKind::JointTree(ref tree) => Some((tree.clone(), true)), - TokenStreamKind::Stream(ref stream) => stream.first().unwrap().first_tree_and_joint(), + match self { + TokenStream::Empty => None, + TokenStream::Tree(ref tree) => Some((tree.clone(), false)), + TokenStream::JointTree(ref tree) => Some((tree.clone(), true)), + TokenStream::Stream(ref stream) => stream.first().unwrap().first_tree_and_joint(), } } fn last_tree_if_joint(&self) -> Option { - match self.kind { - TokenStreamKind::Empty | TokenStreamKind::Tree(..) => None, - TokenStreamKind::JointTree(ref tree) => Some(tree.clone()), - TokenStreamKind::Stream(ref stream) => stream.last().unwrap().last_tree_if_joint(), + match self { + TokenStream::Empty | TokenStream::Tree(..) => None, + TokenStream::JointTree(ref tree) => Some(tree.clone()), + TokenStream::Stream(ref stream) => stream.last().unwrap().last_tree_if_joint(), } } } @@ -472,7 +465,7 @@ impl TokenStreamBuilder { } fn push_all_but_last_tree(&mut self, stream: &TokenStream) { - if let TokenStreamKind::Stream(ref streams) = stream.kind { + if let TokenStream::Stream(ref streams) = stream { let len = streams.len(); match len { 1 => {} @@ -484,7 +477,7 @@ impl TokenStreamBuilder { } fn push_all_but_first_tree(&mut self, stream: &TokenStream) { - if let TokenStreamKind::Stream(ref streams) = stream.kind { + if let TokenStream::Stream(ref streams) = stream { let len = streams.len(); match len { 1 => {} @@ -524,10 +517,10 @@ impl StreamCursor { if self.index < self.stream.len() { self.index += 1; let next = self.stream[self.index - 1].clone(); - match next.kind { - TokenStreamKind::Tree(..) | TokenStreamKind::JointTree(..) => return Some(next), - TokenStreamKind::Stream(stream) => self.insert(stream), - TokenStreamKind::Empty => {} + match next { + TokenStream::Tree(..) | TokenStream::JointTree(..) => return Some(next), + TokenStream::Stream(stream) => self.insert(stream), + TokenStream::Empty => {} } } else if let Some((stream, index)) = self.stack.pop() { self.stream = stream; @@ -548,8 +541,8 @@ impl Iterator for Cursor { type Item = TokenTree; fn next(&mut self) -> Option { - self.next_as_stream().map(|stream| match stream.kind { - TokenStreamKind::Tree(tree) | TokenStreamKind::JointTree(tree) => tree, + self.next_as_stream().map(|stream| match stream { + TokenStream::Tree(tree) | TokenStream::JointTree(tree) => tree, _ => unreachable!() }) } @@ -557,11 +550,11 @@ impl Iterator for Cursor { impl Cursor { fn new(stream: TokenStream) -> Self { - Cursor(match stream.kind { - TokenStreamKind::Empty => CursorKind::Empty, - TokenStreamKind::Tree(tree) => CursorKind::Tree(tree, false), - TokenStreamKind::JointTree(tree) => CursorKind::JointTree(tree, false), - TokenStreamKind::Stream(stream) => CursorKind::Stream(StreamCursor::new(stream)), + Cursor(match stream { + TokenStream::Empty => CursorKind::Empty, + TokenStream::Tree(tree) => CursorKind::Tree(tree, false), + TokenStream::JointTree(tree) => CursorKind::JointTree(tree, false), + TokenStream::Stream(stream) => CursorKind::Stream(StreamCursor::new(stream)), }) } @@ -610,11 +603,11 @@ impl Cursor { pub fn look_ahead(&self, n: usize) -> Option { fn look_ahead(streams: &[TokenStream], mut n: usize) -> Result { for stream in streams { - n = match stream.kind { - TokenStreamKind::Tree(ref tree) | TokenStreamKind::JointTree(ref tree) + n = match stream { + TokenStream::Tree(ref tree) | TokenStream::JointTree(ref tree) if n == 0 => return Ok(tree.clone()), - TokenStreamKind::Tree(..) | TokenStreamKind::JointTree(..) => n - 1, - TokenStreamKind::Stream(ref stream) => match look_ahead(stream, n) { + TokenStream::Tree(..) | TokenStream::JointTree(..) => n - 1, + TokenStream::Stream(ref stream) => match look_ahead(stream, n) { Ok(tree) => return Ok(tree), Err(n) => n, }, @@ -660,11 +653,11 @@ impl ThinTokenStream { impl From for ThinTokenStream { fn from(stream: TokenStream) -> ThinTokenStream { - ThinTokenStream(match stream.kind { - TokenStreamKind::Empty => None, - TokenStreamKind::Tree(tree) => Some(Lrc::new(vec![tree.into()])), - TokenStreamKind::JointTree(tree) => Some(Lrc::new(vec![tree.joint()])), - TokenStreamKind::Stream(stream) => Some(stream), + ThinTokenStream(match stream { + TokenStream::Empty => None, + TokenStream::Tree(tree) => Some(Lrc::new(vec![tree.into()])), + TokenStream::JointTree(tree) => Some(Lrc::new(vec![tree.joint()])), + TokenStream::Stream(stream) => Some(stream), }) } } From e80c7ddb05ee584c12131a4713173a3eafc49f4a Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 12 Dec 2018 10:01:08 +1100 Subject: [PATCH 6/6] Rename `TokenStream::concat` and remove `TokenStream::concat_rc_vec`. `TokenStream::new` is a better name for the former, and the latter is now just equivalent to `TokenStream::Stream`. --- src/libsyntax/attr/mod.rs | 6 ++--- src/libsyntax/ext/quote.rs | 2 +- src/libsyntax/ext/tt/transcribe.rs | 4 +-- src/libsyntax/parse/attr.rs | 2 +- src/libsyntax/parse/lexer/tokentrees.rs | 6 ++--- src/libsyntax/parse/mod.rs | 6 ++--- src/libsyntax/parse/parser.rs | 4 +-- src/libsyntax/tokenstream.rs | 34 +++++++++++-------------- 8 files changed, 30 insertions(+), 34 deletions(-) diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs index 7723c15a266f1..73cbe49f43b04 100644 --- a/src/libsyntax/attr/mod.rs +++ b/src/libsyntax/attr/mod.rs @@ -483,7 +483,7 @@ impl MetaItem { last_pos = segment.ident.span.hi(); } idents.push(self.node.tokens(self.span)); - TokenStream::concat(idents) + TokenStream::new(idents) } fn from_tokens(tokens: &mut iter::Peekable) -> Option @@ -539,7 +539,7 @@ impl MetaItemKind { match *self { MetaItemKind::Word => TokenStream::empty(), MetaItemKind::NameValue(ref lit) => { - TokenStream::concat(vec![TokenTree::Token(span, Token::Eq).into(), lit.tokens()]) + TokenStream::new(vec![TokenTree::Token(span, Token::Eq).into(), lit.tokens()]) } MetaItemKind::List(ref list) => { let mut tokens = Vec::new(); @@ -552,7 +552,7 @@ impl MetaItemKind { TokenTree::Delimited( DelimSpan::from_single(span), token::Paren, - TokenStream::concat(tokens).into(), + TokenStream::new(tokens).into(), ).into() } } diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs index c3497a17806b1..5820b49ab6216 100644 --- a/src/libsyntax/ext/quote.rs +++ b/src/libsyntax/ext/quote.rs @@ -247,7 +247,7 @@ pub mod rt { let delim_span = DelimSpan::from_single(self.span); r.push(TokenTree::Delimited( - delim_span, token::Bracket, TokenStream::concat(inner).into() + delim_span, token::Bracket, TokenStream::new(inner).into() )); r } diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs index a76779ffebdc0..a63abd4049513 100644 --- a/src/libsyntax/ext/tt/transcribe.rs +++ b/src/libsyntax/ext/tt/transcribe.rs @@ -103,12 +103,12 @@ pub fn transcribe(cx: &ExtCtxt, } Frame::Delimited { forest, span, .. } => { if result_stack.is_empty() { - return TokenStream::concat(result); + return TokenStream::new(result); } let tree = TokenTree::Delimited( span, forest.delim, - TokenStream::concat(result).into(), + TokenStream::new(result).into(), ); result = result_stack.pop().unwrap(); result.push(tree.into()); diff --git a/src/libsyntax/parse/attr.rs b/src/libsyntax/parse/attr.rs index 4ff6048e82178..1bd0656846bce 100644 --- a/src/libsyntax/parse/attr.rs +++ b/src/libsyntax/parse/attr.rs @@ -170,7 +170,7 @@ impl<'a> Parser<'a> { token::CloseDelim(_) | token::Eof => self.unexpected()?, _ => self.parse_token_tree(), }; - TokenStream::concat(vec![eq.into(), tree.into()]) + TokenStream::new(vec![eq.into(), tree.into()]) } else { TokenStream::empty() }; diff --git a/src/libsyntax/parse/lexer/tokentrees.rs b/src/libsyntax/parse/lexer/tokentrees.rs index 86c87cf898d09..0906c25cab361 100644 --- a/src/libsyntax/parse/lexer/tokentrees.rs +++ b/src/libsyntax/parse/lexer/tokentrees.rs @@ -22,7 +22,7 @@ impl<'a> StringReader<'a> { tts.push(self.parse_token_tree()?); } - Ok(TokenStream::concat(tts)) + Ok(TokenStream::new(tts)) } // Parse a stream of tokens into a list of `TokenTree`s, up to a `CloseDelim`. @@ -30,14 +30,14 @@ impl<'a> StringReader<'a> { let mut tts = vec![]; loop { if let token::CloseDelim(..) = self.token { - return TokenStream::concat(tts); + return TokenStream::new(tts); } match self.parse_token_tree() { Ok(tree) => tts.push(tree), Err(mut e) => { e.emit(); - return TokenStream::concat(tts); + return TokenStream::new(tts); } } } diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index eb71003d3d0cf..a1685d537c8bb 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -842,13 +842,13 @@ mod tests { with_globals(|| { let tts = string_to_stream("fn a (b : i32) { b; }".to_string()); - let expected = TokenStream::concat(vec![ + let expected = TokenStream::new(vec![ TokenTree::Token(sp(0, 2), token::Ident(Ident::from_str("fn"), false)).into(), TokenTree::Token(sp(3, 4), token::Ident(Ident::from_str("a"), false)).into(), TokenTree::Delimited( DelimSpan::from_pair(sp(5, 6), sp(13, 14)), token::DelimToken::Paren, - TokenStream::concat(vec![ + TokenStream::new(vec![ TokenTree::Token(sp(6, 7), token::Ident(Ident::from_str("b"), false)).into(), TokenTree::Token(sp(8, 9), token::Colon).into(), @@ -859,7 +859,7 @@ mod tests { TokenTree::Delimited( DelimSpan::from_pair(sp(15, 16), sp(20, 21)), token::DelimToken::Brace, - TokenStream::concat(vec![ + TokenStream::new(vec![ TokenTree::Token(sp(17, 18), token::Ident(Ident::from_str("b"), false)).into(), TokenTree::Token(sp(18, 19), token::Semi).into(), diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index ded6da9f3adb8..4e209f580248d 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -2928,7 +2928,7 @@ impl<'a> Parser<'a> { _ => result.push(self.parse_token_tree().into()), } } - TokenStream::concat(result) + TokenStream::new(result) } /// Parse a prefix-unary-operator expr @@ -4624,7 +4624,7 @@ impl<'a> Parser<'a> { self.unexpected()?; unreachable!() }; - TokenStream::concat(vec![ + TokenStream::new(vec![ args.into(), TokenTree::Token(token_lo.to(self.prev_span), token::FatArrow).into(), body.into(), diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index e31bde4082dad..c11ef33f931d8 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -195,7 +195,7 @@ impl TokenStream { new_stream.extend_from_slice(parts.0); new_stream.push(comma); new_stream.extend_from_slice(parts.1); - return Some((TokenStream::concat(new_stream), sp)); + return Some((TokenStream::new(new_stream), sp)); } } None @@ -216,7 +216,7 @@ impl From for TokenStream { impl> iter::FromIterator for TokenStream { fn from_iter>(iter: I) -> Self { - TokenStream::concat(iter.into_iter().map(Into::into).collect::>()) + TokenStream::new(iter.into_iter().map(Into::into).collect::>()) } } @@ -265,7 +265,7 @@ impl Extend for TokenStream { // Build the resulting token stream. If it contains more than one token, // preserve capacity in the vector in anticipation of the caller // performing additional calls to extend. - *self = TokenStream::concat(builder.0); + *self = TokenStream::new(builder.0); } } @@ -297,18 +297,14 @@ impl TokenStream { } } - pub fn concat(mut streams: Vec) -> TokenStream { + pub fn new(mut streams: Vec) -> TokenStream { match streams.len() { 0 => TokenStream::empty(), 1 => streams.pop().unwrap(), - _ => TokenStream::concat_rc_vec(Lrc::new(streams)), + _ => TokenStream::Stream(Lrc::new(streams)), } } - fn concat_rc_vec(streams: Lrc>) -> TokenStream { - TokenStream::Stream(streams) - } - pub fn trees(&self) -> Cursor { self.clone().into_trees() } @@ -389,7 +385,7 @@ impl TokenStream { }); i += 1; } - TokenStream::concat(result) + TokenStream::new(result) } pub fn map TokenTree>(self, mut f: F) -> TokenStream { @@ -402,7 +398,7 @@ impl TokenStream { _ => unreachable!() }); } - TokenStream::concat(result) + TokenStream::new(result) } fn first_tree_and_joint(&self) -> Option<(TokenTree, bool)> { @@ -461,7 +457,7 @@ impl TokenStreamBuilder { } pub fn build(self) -> TokenStream { - TokenStream::concat(self.0) + TokenStream::new(self.0) } fn push_all_but_last_tree(&mut self, stream: &TokenStream) { @@ -470,7 +466,7 @@ impl TokenStreamBuilder { match len { 1 => {} 2 => self.0.push(streams[0].clone().into()), - _ => self.0.push(TokenStream::concat(streams[0 .. len - 1].to_vec())), + _ => self.0.push(TokenStream::new(streams[0 .. len - 1].to_vec())), } self.push_all_but_last_tree(&streams[len - 1]) } @@ -482,7 +478,7 @@ impl TokenStreamBuilder { match len { 1 => {} 2 => self.0.push(streams[1].clone().into()), - _ => self.0.push(TokenStream::concat(streams[1 .. len].to_vec())), + _ => self.0.push(TokenStream::new(streams[1 .. len].to_vec())), } self.push_all_but_first_tree(&streams[0]) } @@ -577,7 +573,7 @@ impl Cursor { _ if stream.is_empty() => return, CursorKind::Empty => *self = stream.trees(), CursorKind::Tree(_, consumed) | CursorKind::JointTree(_, consumed) => { - *self = TokenStream::concat(vec![self.original_stream(), stream]).trees(); + *self = TokenStream::new(vec![self.original_stream(), stream]).trees(); if consumed { self.next(); } @@ -593,10 +589,10 @@ impl Cursor { CursorKind::Empty => TokenStream::empty(), CursorKind::Tree(ref tree, _) => tree.clone().into(), CursorKind::JointTree(ref tree, _) => tree.clone().joint(), - CursorKind::Stream(ref cursor) => TokenStream::concat_rc_vec({ + CursorKind::Stream(ref cursor) => TokenStream::Stream( cursor.stack.get(0).cloned().map(|(stream, _)| stream) .unwrap_or_else(|| cursor.stream.clone()) - }), + ), } } @@ -664,7 +660,7 @@ impl From for ThinTokenStream { impl From for TokenStream { fn from(stream: ThinTokenStream) -> TokenStream { - stream.0.map(TokenStream::concat_rc_vec).unwrap_or_else(TokenStream::empty) + stream.0.map(TokenStream::Stream).unwrap_or_else(TokenStream::empty) } } @@ -763,7 +759,7 @@ mod tests { let test_res = string_to_ts("foo::bar::baz"); let test_fst = string_to_ts("foo::bar"); let test_snd = string_to_ts("::baz"); - let eq_res = TokenStream::concat(vec![test_fst, test_snd]); + let eq_res = TokenStream::new(vec![test_fst, test_snd]); assert_eq!(test_res.trees().count(), 5); assert_eq!(eq_res.trees().count(), 5); assert_eq!(test_res.eq_unspanned(&eq_res), true);