diff --git a/src/librustdoc/html/highlight.rs b/src/librustdoc/html/highlight.rs index 281bd72deeb80..99ca8c43cfbe2 100644 --- a/src/librustdoc/html/highlight.rs +++ b/src/librustdoc/html/highlight.rs @@ -257,7 +257,7 @@ impl<'a> Classifier<'a> { token::Question => Class::QuestionMark, token::Dollar => { - if self.lexer.peek().kind.is_ident() { + if self.lexer.peek().is_ident() { self.in_macro_nonterminal = true; Class::MacroNonTerminal } else { diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs index b5d9b761773b4..d7e43f645df7b 100644 --- a/src/libsyntax/attr/mod.rs +++ b/src/libsyntax/attr/mod.rs @@ -20,7 +20,7 @@ use crate::source_map::{BytePos, Spanned, dummy_spanned}; use crate::parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration}; use crate::parse::parser::Parser; use crate::parse::{self, ParseSess, PResult}; -use crate::parse::token::{self, Token, TokenKind}; +use crate::parse::token::{self, Token}; use crate::ptr::P; use crate::symbol::{sym, Symbol}; use crate::ThinVec; @@ -467,8 +467,7 @@ impl MetaItem { segment.ident.span.ctxt()); idents.push(TokenTree::token(token::ModSep, mod_sep_span).into()); } - idents.push(TokenTree::token(TokenKind::from_ast_ident(segment.ident), - segment.ident.span).into()); + idents.push(TokenTree::Token(Token::from_ast_ident(segment.ident)).into()); last_pos = segment.ident.span.hi(); } self.node.tokens(self.span).append_to_tree_and_joint_vec(&mut idents); diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs index 1c7fdf995e15a..9520adb9029c2 100644 --- a/src/libsyntax/ext/tt/macro_parser.rs +++ b/src/libsyntax/ext/tt/macro_parser.rs @@ -428,13 +428,13 @@ pub fn parse_failure_msg(tok: TokenKind) -> String { } /// Performs a token equality check, ignoring syntax context (that is, an unhygienic comparison) -fn token_name_eq(t1: &TokenKind, t2: &TokenKind) -> bool { - if let (Some((name1, is_raw1)), Some((name2, is_raw2))) = (t1.ident_name(), t2.ident_name()) { - name1 == name2 && is_raw1 == is_raw2 - } else if let (Some(name1), Some(name2)) = (t1.lifetime_name(), t2.lifetime_name()) { - name1 == name2 +fn token_name_eq(t1: &Token, t2: &Token) -> bool { + if let (Some((ident1, is_raw1)), Some((ident2, is_raw2))) = (t1.ident(), t2.ident()) { + ident1.name == ident2.name && is_raw1 == is_raw2 + } else if let (Some(ident1), Some(ident2)) = (t1.lifetime(), t2.lifetime()) { + ident1.name == ident2.name } else { - *t1 == *t2 + t1.kind == t2.kind } } @@ -712,7 +712,7 @@ pub fn parse( // If we reached the EOF, check that there is EXACTLY ONE possible matcher. Otherwise, // either the parse is ambiguous (which should never happen) or there is a syntax error. - if token_name_eq(&parser.token, &token::Eof) { + if parser.token == token::Eof { if eof_items.len() == 1 { let matches = eof_items[0] .matches diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs index 3408cefdf4274..c51f4b20c31c0 100644 --- a/src/libsyntax/ext/tt/transcribe.rs +++ b/src/libsyntax/ext/tt/transcribe.rs @@ -4,7 +4,7 @@ use crate::ext::expand::Marker; use crate::ext::tt::macro_parser::{MatchedNonterminal, MatchedSeq, NamedMatch}; use crate::ext::tt::quoted; use crate::mut_visit::noop_visit_tt; -use crate::parse::token::{self, NtTT, Token, TokenKind}; +use crate::parse::token::{self, NtTT, Token}; use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint}; use smallvec::{smallvec, SmallVec}; @@ -237,7 +237,7 @@ pub fn transcribe( Ident::new(ident.name, ident.span.apply_mark(cx.current_expansion.mark)); sp = sp.apply_mark(cx.current_expansion.mark); result.push(TokenTree::token(token::Dollar, sp).into()); - result.push(TokenTree::token(TokenKind::from_ast_ident(ident), sp).into()); + result.push(TokenTree::Token(Token::from_ast_ident(ident)).into()); } } diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs index e3d959c2c54c4..2f4c48d4bf9e0 100644 --- a/src/libsyntax/parse/lexer/mod.rs +++ b/src/libsyntax/parse/lexer/mod.rs @@ -1501,7 +1501,7 @@ fn char_at(s: &str, byte: usize) -> char { mod tests { use super::*; - use crate::ast::{Ident, CrateConfig}; + use crate::ast::CrateConfig; use crate::symbol::Symbol; use crate::source_map::{SourceMap, FilePathMapping}; use crate::feature_gate::UnstableFeatures; @@ -1562,7 +1562,7 @@ mod tests { assert_eq!(string_reader.next_token(), token::Whitespace); let tok1 = string_reader.next_token(); let tok2 = Token::new( - token::Ident(Symbol::intern("fn"), false), + mk_ident("fn"), Span::new(BytePos(21), BytePos(23), NO_EXPANSION), ); assert_eq!(tok1.kind, tok2.kind); @@ -1593,7 +1593,7 @@ mod tests { // make the identifier by looking up the string in the interner fn mk_ident(id: &str) -> TokenKind { - TokenKind::from_ast_ident(Ident::from_str(id)) + token::Ident(Symbol::intern(id), false) } fn mk_lit(kind: token::LitKind, symbol: &str, suffix: Option<&str>) -> TokenKind { diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 3acd708814560..8ad43a6c809bd 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -2627,9 +2627,11 @@ impl<'a> Parser<'a> { token::Ident(name, _) => name, _ => unreachable!() }; - let mut err = self.fatal(&format!("unknown macro variable `{}`", name)); - err.span_label(self.token.span, "unknown macro variable"); - err.emit(); + let span = self.prev_span.to(self.token.span); + self.diagnostic() + .struct_span_fatal(span, &format!("unknown macro variable `{}`", name)) + .span_label(span, "unknown macro variable") + .emit(); self.bump(); return } diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 28a733728bf7b..0ece5b57935e5 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -241,21 +241,19 @@ pub struct Token { pub span: Span, } -impl TokenKind { - /// Recovers a `TokenKind` from an `ast::Ident`. This creates a raw identifier if necessary. - pub fn from_ast_ident(ident: ast::Ident) -> TokenKind { - Ident(ident.name, ident.is_raw_guess()) +impl Token { + /// Recovers a `Token` from an `ast::Ident`. This creates a raw identifier if necessary. + crate fn from_ast_ident(ident: ast::Ident) -> Token { + Token::new(Ident(ident.name, ident.is_raw_guess()), ident.span) } crate fn is_like_plus(&self) -> bool { - match *self { + match self.kind { BinOp(Plus) | BinOpEq(Plus) => true, _ => false, } } -} -impl Token { /// Returns `true` if the token can appear at the start of an expression. crate fn can_begin_expr(&self) -> bool { match self.kind { @@ -310,12 +308,10 @@ impl Token { _ => false, } } -} -impl TokenKind { /// Returns `true` if the token can appear at the start of a const param. - pub fn can_begin_const_arg(&self) -> bool { - match self { + crate fn can_begin_const_arg(&self) -> bool { + match self.kind { OpenDelim(Brace) => true, Interpolated(ref nt) => match **nt { NtExpr(..) => true, @@ -326,9 +322,7 @@ impl TokenKind { _ => self.can_begin_literal_or_bool(), } } -} -impl Token { /// Returns `true` if the token can appear at the start of a generic bound. crate fn can_begin_bound(&self) -> bool { self.is_path_start() || self.is_lifetime() || self.is_keyword(kw::For) || @@ -340,17 +334,19 @@ impl TokenKind { pub fn lit(kind: LitKind, symbol: Symbol, suffix: Option) -> TokenKind { Literal(Lit::new(kind, symbol, suffix)) } +} +impl Token { /// Returns `true` if the token is any literal crate fn is_lit(&self) -> bool { - match *self { + match self.kind { Literal(..) => true, _ => false, } } crate fn expect_lit(&self) -> Lit { - match *self { + match self.kind { Literal(lit) => lit, _=> panic!("`expect_lit` called on non-literal"), } @@ -359,7 +355,7 @@ impl TokenKind { /// Returns `true` if the token is any literal, a minus (which can prefix a literal, /// for example a '-42', or one of the boolean idents). crate fn can_begin_literal_or_bool(&self) -> bool { - match *self { + match self.kind { Literal(..) => true, BinOp(Minus) => true, Ident(name, false) if name == kw::True => true, @@ -371,9 +367,7 @@ impl TokenKind { _ => false, } } -} -impl Token { /// Returns an identifier if this token is an identifier. pub fn ident(&self) -> Option<(ast::Ident, /* is_raw */ bool)> { match self.kind { @@ -397,49 +391,25 @@ impl Token { _ => None, } } -} -impl TokenKind { - /// Returns an identifier name if this token is an identifier. - pub fn ident_name(&self) -> Option<(ast::Name, /* is_raw */ bool)> { - match *self { - Ident(name, is_raw) => Some((name, is_raw)), - Interpolated(ref nt) => match **nt { - NtIdent(ident, is_raw) => Some((ident.name, is_raw)), - _ => None, - }, - _ => None, - } - } - /// Returns a lifetime name if this token is a lifetime. - pub fn lifetime_name(&self) -> Option { - match *self { - Lifetime(name) => Some(name), - Interpolated(ref nt) => match **nt { - NtLifetime(ident) => Some(ident.name), - _ => None, - }, - _ => None, - } - } /// Returns `true` if the token is an identifier. pub fn is_ident(&self) -> bool { - self.ident_name().is_some() + self.ident().is_some() } /// Returns `true` if the token is a lifetime. crate fn is_lifetime(&self) -> bool { - self.lifetime_name().is_some() + self.lifetime().is_some() } /// Returns `true` if the token is a identifier whose name is the given /// string slice. crate fn is_ident_named(&self, name: Symbol) -> bool { - self.ident_name().map_or(false, |(ident_name, _)| ident_name == name) + self.ident().map_or(false, |(ident, _)| ident.name == name) } /// Returns `true` if the token is an interpolated path. fn is_path(&self) -> bool { - if let Interpolated(ref nt) = *self { + if let Interpolated(ref nt) = self.kind { if let NtPath(..) = **nt { return true; } @@ -456,33 +426,27 @@ impl TokenKind { crate fn is_qpath_start(&self) -> bool { self == &Lt || self == &BinOp(Shl) } -} -impl Token { crate fn is_path_start(&self) -> bool { self == &ModSep || self.is_qpath_start() || self.is_path() || self.is_path_segment_keyword() || self.is_ident() && !self.is_reserved_ident() } -} -impl TokenKind { /// Returns `true` if the token is a given keyword, `kw`. pub fn is_keyword(&self, kw: Symbol) -> bool { - self.ident_name().map(|(name, is_raw)| name == kw && !is_raw).unwrap_or(false) + self.ident().map(|(id, is_raw)| id.name == kw && !is_raw).unwrap_or(false) } - pub fn is_path_segment_keyword(&self) -> bool { - match self.ident_name() { - Some((name, false)) => name.is_path_segment_keyword(), + crate fn is_path_segment_keyword(&self) -> bool { + match self.ident() { + Some((id, false)) => id.is_path_segment_keyword(), _ => false, } } -} -impl Token { // Returns true for reserved identifiers used internally for elided lifetimes, // unnamed method parameters, crate root module, error recovery etc. - pub fn is_special_ident(&self) -> bool { + crate fn is_special_ident(&self) -> bool { match self.ident() { Some((id, false)) => id.is_special(), _ => false, @@ -512,55 +476,53 @@ impl Token { _ => false, } } -} -impl TokenKind { - crate fn glue(self, joint: TokenKind) -> Option { - Some(match self { - Eq => match joint { + crate fn glue(self, joint: Token) -> Option { + let kind = match self.kind { + Eq => match joint.kind { Eq => EqEq, Gt => FatArrow, _ => return None, }, - Lt => match joint { + Lt => match joint.kind { Eq => Le, Lt => BinOp(Shl), Le => BinOpEq(Shl), BinOp(Minus) => LArrow, _ => return None, }, - Gt => match joint { + Gt => match joint.kind { Eq => Ge, Gt => BinOp(Shr), Ge => BinOpEq(Shr), _ => return None, }, - Not => match joint { + Not => match joint.kind { Eq => Ne, _ => return None, }, - BinOp(op) => match joint { + BinOp(op) => match joint.kind { Eq => BinOpEq(op), BinOp(And) if op == And => AndAnd, BinOp(Or) if op == Or => OrOr, Gt if op == Minus => RArrow, _ => return None, }, - Dot => match joint { + Dot => match joint.kind { Dot => DotDot, DotDot => DotDotDot, _ => return None, }, - DotDot => match joint { + DotDot => match joint.kind { Dot => DotDotDot, Eq => DotDotEq, _ => return None, }, - Colon => match joint { + Colon => match joint.kind { Colon => ModSep, _ => return None, }, - SingleQuote => match joint { + SingleQuote => match joint.kind { Ident(name, false) => Lifetime(Symbol::intern(&format!("'{}", name))), _ => return None, }, @@ -570,9 +532,13 @@ impl TokenKind { Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..) | Lifetime(..) | Interpolated(..) | DocComment(..) | Whitespace | Comment | Shebang(..) | Eof => return None, - }) + }; + + Some(Token::new(kind, self.span.to(joint.span))) } +} +impl TokenKind { /// Returns tokens that are likely to be typed accidentally instead of the current token. /// Enables better error recovery when the wrong token is found. crate fn similar_tokens(&self) -> Option> { @@ -582,14 +548,16 @@ impl TokenKind { _ => None } } +} +impl Token { // See comments in `Nonterminal::to_tokenstream` for why we care about // *probably* equal here rather than actual equality crate fn probably_equal_for_proc_macro(&self, other: &TokenKind) -> bool { - if mem::discriminant(self) != mem::discriminant(other) { + if mem::discriminant(&self.kind) != mem::discriminant(other) { return false } - match (self, other) { + match (&self.kind, other) { (&Eq, &Eq) | (&Lt, &Lt) | (&Le, &Le) | @@ -641,9 +609,7 @@ impl TokenKind { _ => panic!("forgot to add a token?"), } } -} -impl Token { crate fn new(kind: TokenKind, span: Span) -> Self { Token { kind, span } } diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index 9dea3a4dcc144..d46d2f549c09b 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -430,11 +430,10 @@ impl TokenStreamBuilder { let last_tree_if_joint = self.0.last().and_then(TokenStream::last_tree_if_joint); if let Some(TokenTree::Token(last_token)) = last_tree_if_joint { if let Some((TokenTree::Token(token), is_joint)) = stream.first_tree_and_joint() { - if let Some(glued_tok) = last_token.kind.glue(token.kind) { + if let Some(glued_tok) = last_token.glue(token) { let last_stream = self.0.pop().unwrap(); self.push_all_but_last_tree(&last_stream); - let glued_span = last_token.span.to(token.span); - let glued_tt = TokenTree::token(glued_tok, glued_span); + let glued_tt = TokenTree::Token(glued_tok); let glued_tokenstream = TokenStream::new(vec![(glued_tt, is_joint)]); self.0.push(glued_tokenstream); self.push_all_but_first_tree(&stream); diff --git a/src/libsyntax/util/parser.rs b/src/libsyntax/util/parser.rs index 9e26f1bf7d374..69dd96625cc02 100644 --- a/src/libsyntax/util/parser.rs +++ b/src/libsyntax/util/parser.rs @@ -1,4 +1,4 @@ -use crate::parse::token::{self, TokenKind, BinOpToken}; +use crate::parse::token::{self, Token, BinOpToken}; use crate::symbol::kw; use crate::ast::{self, BinOpKind}; @@ -69,9 +69,9 @@ pub enum Fixity { impl AssocOp { /// Creates a new AssocOP from a token - pub fn from_token(t: &TokenKind) -> Option { + pub fn from_token(t: &Token) -> Option { use AssocOp::*; - match *t { + match t.kind { token::BinOpEq(k) => Some(AssignOp(k)), token::Eq => Some(Assign), token::BinOp(BinOpToken::Star) => Some(Multiply),