diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index e76605cde32ab..47da3ee6a6c78 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -62,19 +62,7 @@ pub struct StringReader<'a> { // cache a direct reference to the source text, so that we don't have to // retrieve it via `self.source_file.src.as_ref().unwrap()` all the time. src: Lrc, - token: token::Token, - span: Span, - /// The raw source span which *does not* take `override_span` into account - span_src_raw: Span, - /// Stack of open delimiters and their spans. Used for error message. - open_braces: Vec<(token::DelimToken, Span)>, - crate unmatched_braces: Vec, - /// The type and spans for all braces - /// - /// Used only for error recovery when arriving to EOF with mismatched braces. - matching_delim_spans: Vec<(token::DelimToken, Span, Span)>, - crate override_span: Option, - last_unclosed_found_span: Option, + override_span: Option, } impl<'a> StringReader<'a> { @@ -121,8 +109,6 @@ impl<'a> StringReader<'a> { sp: self.peek_span, }; self.advance_token()?; - self.span_src_raw = self.peek_span_src_raw; - Ok(ret_val) } @@ -159,9 +145,6 @@ impl<'a> StringReader<'a> { } } - self.token = t.tok.clone(); - self.span = t.sp; - Ok(t) } @@ -251,14 +234,7 @@ impl<'a> StringReader<'a> { peek_span_src_raw: syntax_pos::DUMMY_SP, src, fatal_errs: Vec::new(), - token: token::Eof, - span: syntax_pos::DUMMY_SP, - span_src_raw: syntax_pos::DUMMY_SP, - open_braces: Vec::new(), - unmatched_braces: Vec::new(), - matching_delim_spans: Vec::new(), override_span, - last_unclosed_found_span: None, } } diff --git a/src/libsyntax/parse/lexer/tokentrees.rs b/src/libsyntax/parse/lexer/tokentrees.rs index 0db36c84cdfeb..4bfc5bb16c0bb 100644 --- a/src/libsyntax/parse/lexer/tokentrees.rs +++ b/src/libsyntax/parse/lexer/tokentrees.rs @@ -1,13 +1,46 @@ +use syntax_pos::Span; + use crate::print::pprust::token_to_string; use crate::parse::lexer::{StringReader, UnmatchedBrace}; use crate::parse::{token, PResult}; use crate::tokenstream::{DelimSpan, IsJoint::*, TokenStream, TokenTree, TreeAndJoint}; impl<'a> StringReader<'a> { + crate fn into_token_trees(self) -> (PResult<'a, TokenStream>, Vec) { + let mut tt_reader = TokenTreesReader { + string_reader: self, + token: token::Eof, + span: syntax_pos::DUMMY_SP, + open_braces: Vec::new(), + unmatched_braces: Vec::new(), + matching_delim_spans: Vec::new(), + last_unclosed_found_span: None, + }; + let res = tt_reader.parse_all_token_trees(); + (res, tt_reader.unmatched_braces) + } +} + +struct TokenTreesReader<'a> { + string_reader: StringReader<'a>, + token: token::Token, + span: Span, + /// Stack of open delimiters and their spans. Used for error message. + open_braces: Vec<(token::DelimToken, Span)>, + unmatched_braces: Vec, + /// The type and spans for all braces + /// + /// Used only for error recovery when arriving to EOF with mismatched braces. + matching_delim_spans: Vec<(token::DelimToken, Span, Span)>, + last_unclosed_found_span: Option, +} + +impl<'a> TokenTreesReader<'a> { // Parse a stream of tokens into a list of `TokenTree`s, up to an `Eof`. - crate fn parse_all_token_trees(&mut self) -> PResult<'a, TokenStream> { + fn parse_all_token_trees(&mut self) -> PResult<'a, TokenStream> { let mut tts = Vec::new(); + self.real_token(); while self.token != token::Eof { tts.push(self.parse_token_tree()?); } @@ -34,11 +67,12 @@ impl<'a> StringReader<'a> { } fn parse_token_tree(&mut self) -> PResult<'a, TreeAndJoint> { - let sm = self.sess.source_map(); + let sm = self.string_reader.sess.source_map(); match self.token { token::Eof => { let msg = "this file contains an un-closed delimiter"; - let mut err = self.sess.span_diagnostic.struct_span_err(self.span, msg); + let mut err = self.string_reader.sess.span_diagnostic + .struct_span_err(self.span, msg); for &(_, sp) in &self.open_braces { err.span_label(sp, "un-closed delimiter"); } @@ -46,13 +80,12 @@ impl<'a> StringReader<'a> { if let Some((delim, _)) = self.open_braces.last() { if let Some((_, open_sp, close_sp)) = self.matching_delim_spans.iter() .filter(|(d, open_sp, close_sp)| { - - if let Some(close_padding) = sm.span_to_margin(*close_sp) { - if let Some(open_padding) = sm.span_to_margin(*open_sp) { - return delim == d && close_padding != open_padding; + if let Some(close_padding) = sm.span_to_margin(*close_sp) { + if let Some(open_padding) = sm.span_to_margin(*open_sp) { + return delim == d && close_padding != open_padding; + } } - } - false + false }).next() // these are in reverse order as they get inserted on close, but { // we want the last open/first close err.span_label( @@ -164,7 +197,8 @@ impl<'a> StringReader<'a> { // matching opening delimiter). let token_str = token_to_string(&self.token); let msg = format!("unexpected close delimiter: `{}`", token_str); - let mut err = self.sess.span_diagnostic.struct_span_err(self.span, &msg); + let mut err = self.string_reader.sess.span_diagnostic + .struct_span_err(self.span, &msg); err.span_label(self.span, "unexpected close delimiter"); Err(err) }, @@ -173,11 +207,20 @@ impl<'a> StringReader<'a> { // Note that testing for joint-ness here is done via the raw // source span as the joint-ness is a property of the raw source // rather than wanting to take `override_span` into account. - let raw = self.span_src_raw; + // Additionally, we actually check if the *next* pair of tokens + // is joint, but this is equivalent to checking the current pair. + let raw = self.string_reader.peek_span_src_raw; self.real_token(); - let is_joint = raw.hi() == self.span_src_raw.lo() && token::is_op(&self.token); + let is_joint = raw.hi() == self.string_reader.peek_span_src_raw.lo() + && token::is_op(&self.token); Ok((tt, if is_joint { Joint } else { NonJoint })) } } } + + fn real_token(&mut self) { + let t = self.string_reader.real_token(); + self.token = t.tok; + self.span = t.sp; + } } diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index 655baafcd2d5d..0611c1d9b42a5 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -290,22 +290,22 @@ pub fn source_file_to_stream( } /// Given a source file, produces a sequence of token trees. Returns any buffered errors from -/// parsing the token tream. +/// parsing the token stream. pub fn maybe_file_to_stream( sess: &ParseSess, source_file: Lrc, override_span: Option, ) -> Result<(TokenStream, Vec), Vec> { - let mut srdr = lexer::StringReader::new_or_buffered_errs(sess, source_file, override_span)?; - srdr.real_token(); + let srdr = lexer::StringReader::new_or_buffered_errs(sess, source_file, override_span)?; + let (token_trees, unmatched_braces) = srdr.into_token_trees(); - match srdr.parse_all_token_trees() { - Ok(stream) => Ok((stream, srdr.unmatched_braces)), + match token_trees { + Ok(stream) => Ok((stream, unmatched_braces)), Err(err) => { let mut buffer = Vec::with_capacity(1); err.buffer(&mut buffer); // Not using `emit_unclosed_delims` to use `db.buffer` - for unmatched in srdr.unmatched_braces { + for unmatched in unmatched_braces { let mut db = sess.span_diagnostic.struct_span_err(unmatched.found_span, &format!( "incorrect close delimiter: `{}`", token_to_string(&token::Token::CloseDelim(unmatched.found_delim)),