Remove support for 1-token lookahead from the lexer

src/librustc_save_analysis/span_utils.rs

@@ -53,7 +53,7 @@ impl<'a> SpanUtils<'a> {
     pub fn sub_span_of_token(&self, span: Span, tok: TokenKind) -> Option<Span> {
         let mut toks = self.retokenise_span(span);
         loop {
-            let next = toks.real_token();
+            let next = toks.next_token();
             if next == token::Eof {
                 return None;
             }

src/librustdoc/html/highlight.rs

@@ -38,17 +38,17 @@ pub fn render_with_highlighting(
         FileName::Custom(String::from("rustdoc-highlighting")),
         src.to_owned(),
     );
-    let highlight_result =
-        lexer::StringReader::new_or_buffered_errs(&sess, fm, None).and_then(|lexer| {
-            let mut classifier = Classifier::new(lexer, sess.source_map());
-
-            let mut highlighted_source = vec![];
-            if classifier.write_source(&mut highlighted_source).is_err() {
-                Err(classifier.lexer.buffer_fatal_errors())
-            } else {
-                Ok(String::from_utf8_lossy(&highlighted_source).into_owned())
-            }
-        });
+    let highlight_result = {
+        let lexer = lexer::StringReader::new(&sess, fm, None);
+        let mut classifier = Classifier::new(lexer, sess.source_map());
+
+        let mut highlighted_source = vec![];
+        if classifier.write_source(&mut highlighted_source).is_err() {
+            Err(classifier.lexer.buffer_fatal_errors())
+        } else {
+            Ok(String::from_utf8_lossy(&highlighted_source).into_owned())
+        }
+    };
 
     match highlight_result {
         Ok(highlighted_source) => {
@@ -79,6 +79,7 @@ pub fn render_with_highlighting(
 /// each span of text in sequence.
 struct Classifier<'a> {
     lexer: lexer::StringReader<'a>,
+    peek_token: Option<Token>,
     source_map: &'a SourceMap,
 
     // State of the classifier.
@@ -178,6 +179,7 @@ impl<'a> Classifier<'a> {
     fn new(lexer: lexer::StringReader<'a>, source_map: &'a SourceMap) -> Classifier<'a> {
         Classifier {
             lexer,
+            peek_token: None,
             source_map,
             in_attribute: false,
             in_macro: false,
@@ -187,10 +189,19 @@ impl<'a> Classifier<'a> {
 
     /// Gets the next token out of the lexer.
     fn try_next_token(&mut self) -> Result<Token, HighlightError> {
-        match self.lexer.try_next_token() {
-            Ok(token) => Ok(token),
-            Err(_) => Err(HighlightError::LexError),
+        if let Some(token) = self.peek_token.take() {
+            return Ok(token);
+        }
+        self.lexer.try_next_token().map_err(|()| HighlightError::LexError)
+    }
+
+    fn peek(&mut self) -> Result<&Token, HighlightError> {
+        if self.peek_token.is_none() {
+            self.peek_token = Some(
+                self.lexer.try_next_token().map_err(|()| HighlightError::LexError)?
+            );
         }
+        Ok(self.peek_token.as_ref().unwrap())
     }
 
     /// Exhausts the `lexer` writing the output into `out`.
@@ -234,7 +245,7 @@ impl<'a> Classifier<'a> {
             // reference or dereference operator or a reference or pointer type, instead of the
             // bit-and or multiplication operator.
             token::BinOp(token::And) | token::BinOp(token::Star)
-                if self.lexer.peek() != &token::Whitespace => Class::RefKeyWord,
+                if self.peek()? != &token::Whitespace => Class::RefKeyWord,
 
             // Consider this as part of a macro invocation if there was a
             // leading identifier.
@@ -257,7 +268,7 @@ impl<'a> Classifier<'a> {
             token::Question => Class::QuestionMark,
 
             token::Dollar => {
-                if self.lexer.peek().is_ident() {
+                if self.peek()?.is_ident() {
                     self.in_macro_nonterminal = true;
                     Class::MacroNonTerminal
                 } else {
@@ -280,9 +291,9 @@ impl<'a> Classifier<'a> {
                 // as an attribute.
 
                 // Case 1: #![inner_attribute]
-                if self.lexer.peek() == &token::Not {
+                if self.peek()? == &token::Not {
                     self.try_next_token()?; // NOTE: consumes `!` token!
-                    if self.lexer.peek() == &token::OpenDelim(token::Bracket) {
+                    if self.peek()? == &token::OpenDelim(token::Bracket) {
                         self.in_attribute = true;
                         out.enter_span(Class::Attribute)?;
                     }
@@ -292,7 +303,7 @@ impl<'a> Classifier<'a> {
                 }
 
                 // Case 2: #[outer_attribute]
-                if self.lexer.peek() == &token::OpenDelim(token::Bracket) {
+                if self.peek()? == &token::OpenDelim(token::Bracket) {
                     self.in_attribute = true;
                     out.enter_span(Class::Attribute)?;
                 }
@@ -341,7 +352,7 @@ impl<'a> Classifier<'a> {
                         if self.in_macro_nonterminal {
                             self.in_macro_nonterminal = false;
                             Class::MacroNonTerminal
-                        } else if self.lexer.peek() == &token::Not {
+                        } else if self.peek()? == &token::Not {
                             self.in_macro = true;
                             Class::Macro
                         } else {

src/librustdoc/passes/check_code_block_syntax.rs

@@ -32,7 +32,8 @@ impl<'a, 'tcx> SyntaxChecker<'a, 'tcx> {
             dox[code_block.code].to_owned(),
         );
 
-        let errors = Lexer::new_or_buffered_errs(&sess, source_file, None).and_then(|mut lexer| {
+        let errors = {
+            let mut lexer = Lexer::new(&sess, source_file, None);
             while let Ok(token::Token { kind, .. }) = lexer.try_next_token() {
                 if kind == token::Eof {
                     break;
@@ -46,7 +47,7 @@ impl<'a, 'tcx> SyntaxChecker<'a, 'tcx> {
             } else {
                 Ok(())
             }
-        });
+        };
 
         if let Err(errors) = errors {
             let mut diag = if let Some(sp) =

src/libsyntax/parse/lexer/comments.rs

@@ -268,7 +268,7 @@ fn read_block_comment(rdr: &mut StringReader<'_>,
         while level > 0 {
             debug!("=== block comment level {}", level);
             if rdr.is_eof() {
-                rdr.fatal("unterminated block comment").raise();
+                rdr.fatal_span_(rdr.pos, rdr.pos, "unterminated block comment").raise();
             }
             if rdr.ch_is('\n') {
                 trim_whitespace_prefix_and_push_line(&mut lines, curr_line, col);
@@ -346,7 +346,7 @@ pub fn gather_comments(sess: &ParseSess, path: FileName, srdr: &mut dyn Read) ->
     srdr.read_to_string(&mut src).unwrap();
     let cm = SourceMap::new(sess.source_map().path_mapping().clone());
     let source_file = cm.new_source_file(path, src);
-    let mut rdr = lexer::StringReader::new_raw(sess, source_file, None);
+    let mut rdr = lexer::StringReader::new(sess, source_file, None);
 
     let mut comments: Vec<Comment> = Vec::new();
     let mut code_to_the_left = false; // Only code