From b5e35b128efeed4bfdb4b1ee9d0697389ec9f164 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Tue, 30 Jul 2019 12:33:32 +0300 Subject: [PATCH] remove special code path for unknown tokens --- src/libsyntax/parse/lexer/mod.rs | 73 ++++--------------- src/test/ui/parser/lex-bad-token.rs | 2 + src/test/ui/parser/lex-stray-backslash.rs | 2 + src/test/ui/parser/unicode-quote-chars.rs | 3 + src/test/ui/parser/unicode-quote-chars.stderr | 18 ++++- 5 files changed, 37 insertions(+), 61 deletions(-) diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index 950b1b2ff5340..c209ae1cb9f1f 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -3,7 +3,7 @@ use crate::parse::token::{self, Token, TokenKind}; use crate::symbol::{sym, Symbol}; use crate::parse::unescape_error_reporting::{emit_unescape_error, push_escaped_char}; -use errors::{FatalError, Diagnostic, DiagnosticBuilder}; +use errors::{FatalError, DiagnosticBuilder}; use syntax_pos::{BytePos, Pos, Span, NO_EXPANSION}; use rustc_lexer::Base; use rustc_lexer::unescape; @@ -39,7 +39,6 @@ pub struct StringReader<'a> { pos: BytePos, /// Stop reading src at this index. end_src_index: usize, - fatal_errs: Vec>, /// Source text to tokenize. src: Lrc, override_span: Option, @@ -62,7 +61,6 @@ impl<'a> StringReader<'a> { pos: source_file.start_pos, end_src_index: src.len(), src, - fatal_errs: Vec::new(), override_span, } } @@ -89,29 +87,17 @@ impl<'a> StringReader<'a> { self.override_span.unwrap_or_else(|| Span::new(lo, hi, NO_EXPANSION)) } - fn unwrap_or_abort(&mut self, res: Result) -> Token { - match res { - Ok(tok) => tok, - Err(_) => { - self.emit_fatal_errors(); - FatalError.raise(); - } - } - } - /// Returns the next token, including trivia like whitespace or comments. /// /// `Err(())` means that some errors were encountered, which can be /// retrieved using `buffer_fatal_errors`. - pub fn try_next_token(&mut self) -> Result { - assert!(self.fatal_errs.is_empty()); - + pub fn next_token(&mut self) -> Token { let start_src_index = self.src_index(self.pos); let text: &str = &self.src[start_src_index..self.end_src_index]; if text.is_empty() { let span = self.mk_sp(self.pos, self.pos); - return Ok(Token::new(token::Eof, span)); + return Token::new(token::Eof, span); } { @@ -125,7 +111,7 @@ impl<'a> StringReader<'a> { let kind = token::Shebang(sym); let span = self.mk_sp(start, self.pos); - return Ok(Token::new(kind, span)); + return Token::new(kind, span); } } } @@ -139,39 +125,10 @@ impl<'a> StringReader<'a> { // This could use `?`, but that makes code significantly (10-20%) slower. // https://github.com/rust-lang/rust/issues/37939 - let kind = match self.cook_lexer_token(token.kind, start) { - Ok(it) => it, - Err(err) => return Err(self.fatal_errs.push(err)), - }; + let kind = self.cook_lexer_token(token.kind, start); let span = self.mk_sp(start, self.pos); - Ok(Token::new(kind, span)) - } - - /// Returns the next token, including trivia like whitespace or comments. - /// - /// Aborts in case of an error. - pub fn next_token(&mut self) -> Token { - let res = self.try_next_token(); - self.unwrap_or_abort(res) - } - - fn emit_fatal_errors(&mut self) { - for err in &mut self.fatal_errs { - err.emit(); - } - - self.fatal_errs.clear(); - } - - pub fn buffer_fatal_errors(&mut self) -> Vec { - let mut buffer = Vec::new(); - - for err in self.fatal_errs.drain(..) { - err.buffer(&mut buffer); - } - - buffer + Token::new(kind, span) } /// Report a fatal lexical error with a given span. @@ -218,8 +175,8 @@ impl<'a> StringReader<'a> { &self, token: rustc_lexer::TokenKind, start: BytePos, - ) -> Result> { - let kind = match token { + ) -> TokenKind { + match token { rustc_lexer::TokenKind::LineComment => { let string = self.str_from(start); // comments with only more "/"s are not doc comments @@ -396,16 +353,12 @@ impl<'a> StringReader<'a> { // this should be inside `rustc_lexer`. However, we should first remove compound // tokens like `<<` from `rustc_lexer`, and then add fancier error recovery to it, // as there will be less overall work to do this way. - return match unicode_chars::check_for_substitution(self, start, c, &mut err) { - Some(token) => { - err.emit(); - Ok(token) - } - None => Err(err), - } + let token = unicode_chars::check_for_substitution(self, start, c, &mut err) + .unwrap_or(token::Whitespace); + err.emit(); + token } - }; - Ok(kind) + } } fn cook_lexer_literal( diff --git a/src/test/ui/parser/lex-bad-token.rs b/src/test/ui/parser/lex-bad-token.rs index feb670c3d3dd0..9e4824611128d 100644 --- a/src/test/ui/parser/lex-bad-token.rs +++ b/src/test/ui/parser/lex-bad-token.rs @@ -1 +1,3 @@ ● //~ ERROR: unknown start of token + +fn main() {} diff --git a/src/test/ui/parser/lex-stray-backslash.rs b/src/test/ui/parser/lex-stray-backslash.rs index 90d359231a6e7..bb27f44c279f7 100644 --- a/src/test/ui/parser/lex-stray-backslash.rs +++ b/src/test/ui/parser/lex-stray-backslash.rs @@ -1 +1,3 @@ \ //~ ERROR: unknown start of token: \ + +fn main() {} diff --git a/src/test/ui/parser/unicode-quote-chars.rs b/src/test/ui/parser/unicode-quote-chars.rs index 69644211b8a11..1812dad81afc3 100644 --- a/src/test/ui/parser/unicode-quote-chars.rs +++ b/src/test/ui/parser/unicode-quote-chars.rs @@ -4,4 +4,7 @@ fn main() { println!(“hello world”); //~^ ERROR unknown start of token: \u{201c} //~^^ HELP Unicode characters '“' (Left Double Quotation Mark) and '”' (Right Double Quotation Mark) look like '"' (Quotation Mark), but are not + //~^^^ ERROR unknown start of token: \u{201d} + //~^^^^ HELP Unicode character '”' (Right Double Quotation Mark) looks like '"' (Quotation Mark), but it is not + //~^^^^^ ERROR expected token: `,` } diff --git a/src/test/ui/parser/unicode-quote-chars.stderr b/src/test/ui/parser/unicode-quote-chars.stderr index 4a09ed75605e4..84e45ecd873a4 100644 --- a/src/test/ui/parser/unicode-quote-chars.stderr +++ b/src/test/ui/parser/unicode-quote-chars.stderr @@ -8,5 +8,21 @@ help: Unicode characters '“' (Left Double Quotation Mark) and '”' (Right Dou LL | println!("hello world"); | ^^^^^^^^^^^^^ -error: aborting due to previous error +error: unknown start of token: \u{201d} + --> $DIR/unicode-quote-chars.rs:4:26 + | +LL | println!(“hello world”); + | ^ +help: Unicode character '”' (Right Double Quotation Mark) looks like '"' (Quotation Mark), but it is not + | +LL | println!(“hello world"); + | ^ + +error: expected token: `,` + --> $DIR/unicode-quote-chars.rs:4:21 + | +LL | println!(“hello world”); + | ^^^^^ expected `,` + +error: aborting due to 3 previous errors