diff --git a/src/librustdoc/html/highlight.rs b/src/librustdoc/html/highlight.rs
index 281bd72deeb80..99ca8c43cfbe2 100644
--- a/src/librustdoc/html/highlight.rs
+++ b/src/librustdoc/html/highlight.rs
@@ -257,7 +257,7 @@ impl<'a> Classifier<'a> {
token::Question => Class::QuestionMark,
token::Dollar => {
- if self.lexer.peek().kind.is_ident() {
+ if self.lexer.peek().is_ident() {
self.in_macro_nonterminal = true;
Class::MacroNonTerminal
} else {
diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs
index b5d9b761773b4..d7e43f645df7b 100644
--- a/src/libsyntax/attr/mod.rs
+++ b/src/libsyntax/attr/mod.rs
@@ -20,7 +20,7 @@ use crate::source_map::{BytePos, Spanned, dummy_spanned};
use crate::parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration};
use crate::parse::parser::Parser;
use crate::parse::{self, ParseSess, PResult};
-use crate::parse::token::{self, Token, TokenKind};
+use crate::parse::token::{self, Token};
use crate::ptr::P;
use crate::symbol::{sym, Symbol};
use crate::ThinVec;
@@ -467,8 +467,7 @@ impl MetaItem {
segment.ident.span.ctxt());
idents.push(TokenTree::token(token::ModSep, mod_sep_span).into());
}
- idents.push(TokenTree::token(TokenKind::from_ast_ident(segment.ident),
- segment.ident.span).into());
+ idents.push(TokenTree::Token(Token::from_ast_ident(segment.ident)).into());
last_pos = segment.ident.span.hi();
}
self.node.tokens(self.span).append_to_tree_and_joint_vec(&mut idents);
diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs
index 1c7fdf995e15a..9520adb9029c2 100644
--- a/src/libsyntax/ext/tt/macro_parser.rs
+++ b/src/libsyntax/ext/tt/macro_parser.rs
@@ -428,13 +428,13 @@ pub fn parse_failure_msg(tok: TokenKind) -> String {
}
/// Performs a token equality check, ignoring syntax context (that is, an unhygienic comparison)
-fn token_name_eq(t1: &TokenKind, t2: &TokenKind) -> bool {
- if let (Some((name1, is_raw1)), Some((name2, is_raw2))) = (t1.ident_name(), t2.ident_name()) {
- name1 == name2 && is_raw1 == is_raw2
- } else if let (Some(name1), Some(name2)) = (t1.lifetime_name(), t2.lifetime_name()) {
- name1 == name2
+fn token_name_eq(t1: &Token, t2: &Token) -> bool {
+ if let (Some((ident1, is_raw1)), Some((ident2, is_raw2))) = (t1.ident(), t2.ident()) {
+ ident1.name == ident2.name && is_raw1 == is_raw2
+ } else if let (Some(ident1), Some(ident2)) = (t1.lifetime(), t2.lifetime()) {
+ ident1.name == ident2.name
} else {
- *t1 == *t2
+ t1.kind == t2.kind
}
}
@@ -712,7 +712,7 @@ pub fn parse(
// If we reached the EOF, check that there is EXACTLY ONE possible matcher. Otherwise,
// either the parse is ambiguous (which should never happen) or there is a syntax error.
- if token_name_eq(&parser.token, &token::Eof) {
+ if parser.token == token::Eof {
if eof_items.len() == 1 {
let matches = eof_items[0]
.matches
diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs
index 3408cefdf4274..c51f4b20c31c0 100644
--- a/src/libsyntax/ext/tt/transcribe.rs
+++ b/src/libsyntax/ext/tt/transcribe.rs
@@ -4,7 +4,7 @@ use crate::ext::expand::Marker;
use crate::ext::tt::macro_parser::{MatchedNonterminal, MatchedSeq, NamedMatch};
use crate::ext::tt::quoted;
use crate::mut_visit::noop_visit_tt;
-use crate::parse::token::{self, NtTT, Token, TokenKind};
+use crate::parse::token::{self, NtTT, Token};
use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint};
use smallvec::{smallvec, SmallVec};
@@ -237,7 +237,7 @@ pub fn transcribe(
Ident::new(ident.name, ident.span.apply_mark(cx.current_expansion.mark));
sp = sp.apply_mark(cx.current_expansion.mark);
result.push(TokenTree::token(token::Dollar, sp).into());
- result.push(TokenTree::token(TokenKind::from_ast_ident(ident), sp).into());
+ result.push(TokenTree::Token(Token::from_ast_ident(ident)).into());
}
}
diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs
index e3d959c2c54c4..2f4c48d4bf9e0 100644
--- a/src/libsyntax/parse/lexer/mod.rs
+++ b/src/libsyntax/parse/lexer/mod.rs
@@ -1501,7 +1501,7 @@ fn char_at(s: &str, byte: usize) -> char {
mod tests {
use super::*;
- use crate::ast::{Ident, CrateConfig};
+ use crate::ast::CrateConfig;
use crate::symbol::Symbol;
use crate::source_map::{SourceMap, FilePathMapping};
use crate::feature_gate::UnstableFeatures;
@@ -1562,7 +1562,7 @@ mod tests {
assert_eq!(string_reader.next_token(), token::Whitespace);
let tok1 = string_reader.next_token();
let tok2 = Token::new(
- token::Ident(Symbol::intern("fn"), false),
+ mk_ident("fn"),
Span::new(BytePos(21), BytePos(23), NO_EXPANSION),
);
assert_eq!(tok1.kind, tok2.kind);
@@ -1593,7 +1593,7 @@ mod tests {
// make the identifier by looking up the string in the interner
fn mk_ident(id: &str) -> TokenKind {
- TokenKind::from_ast_ident(Ident::from_str(id))
+ token::Ident(Symbol::intern(id), false)
}
fn mk_lit(kind: token::LitKind, symbol: &str, suffix: Option<&str>) -> TokenKind {
diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs
index 3acd708814560..8ad43a6c809bd 100644
--- a/src/libsyntax/parse/parser.rs
+++ b/src/libsyntax/parse/parser.rs
@@ -2627,9 +2627,11 @@ impl<'a> Parser<'a> {
token::Ident(name, _) => name,
_ => unreachable!()
};
- let mut err = self.fatal(&format!("unknown macro variable `{}`", name));
- err.span_label(self.token.span, "unknown macro variable");
- err.emit();
+ let span = self.prev_span.to(self.token.span);
+ self.diagnostic()
+ .struct_span_fatal(span, &format!("unknown macro variable `{}`", name))
+ .span_label(span, "unknown macro variable")
+ .emit();
self.bump();
return
}
diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index 28a733728bf7b..0ece5b57935e5 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -241,21 +241,19 @@ pub struct Token {
pub span: Span,
}
-impl TokenKind {
- /// Recovers a `TokenKind` from an `ast::Ident`. This creates a raw identifier if necessary.
- pub fn from_ast_ident(ident: ast::Ident) -> TokenKind {
- Ident(ident.name, ident.is_raw_guess())
+impl Token {
+ /// Recovers a `Token` from an `ast::Ident`. This creates a raw identifier if necessary.
+ crate fn from_ast_ident(ident: ast::Ident) -> Token {
+ Token::new(Ident(ident.name, ident.is_raw_guess()), ident.span)
}
crate fn is_like_plus(&self) -> bool {
- match *self {
+ match self.kind {
BinOp(Plus) | BinOpEq(Plus) => true,
_ => false,
}
}
-}
-impl Token {
/// Returns `true` if the token can appear at the start of an expression.
crate fn can_begin_expr(&self) -> bool {
match self.kind {
@@ -310,12 +308,10 @@ impl Token {
_ => false,
}
}
-}
-impl TokenKind {
/// Returns `true` if the token can appear at the start of a const param.
- pub fn can_begin_const_arg(&self) -> bool {
- match self {
+ crate fn can_begin_const_arg(&self) -> bool {
+ match self.kind {
OpenDelim(Brace) => true,
Interpolated(ref nt) => match **nt {
NtExpr(..) => true,
@@ -326,9 +322,7 @@ impl TokenKind {
_ => self.can_begin_literal_or_bool(),
}
}
-}
-impl Token {
/// Returns `true` if the token can appear at the start of a generic bound.
crate fn can_begin_bound(&self) -> bool {
self.is_path_start() || self.is_lifetime() || self.is_keyword(kw::For) ||
@@ -340,17 +334,19 @@ impl TokenKind {
pub fn lit(kind: LitKind, symbol: Symbol, suffix: Option) -> TokenKind {
Literal(Lit::new(kind, symbol, suffix))
}
+}
+impl Token {
/// Returns `true` if the token is any literal
crate fn is_lit(&self) -> bool {
- match *self {
+ match self.kind {
Literal(..) => true,
_ => false,
}
}
crate fn expect_lit(&self) -> Lit {
- match *self {
+ match self.kind {
Literal(lit) => lit,
_=> panic!("`expect_lit` called on non-literal"),
}
@@ -359,7 +355,7 @@ impl TokenKind {
/// Returns `true` if the token is any literal, a minus (which can prefix a literal,
/// for example a '-42', or one of the boolean idents).
crate fn can_begin_literal_or_bool(&self) -> bool {
- match *self {
+ match self.kind {
Literal(..) => true,
BinOp(Minus) => true,
Ident(name, false) if name == kw::True => true,
@@ -371,9 +367,7 @@ impl TokenKind {
_ => false,
}
}
-}
-impl Token {
/// Returns an identifier if this token is an identifier.
pub fn ident(&self) -> Option<(ast::Ident, /* is_raw */ bool)> {
match self.kind {
@@ -397,49 +391,25 @@ impl Token {
_ => None,
}
}
-}
-impl TokenKind {
- /// Returns an identifier name if this token is an identifier.
- pub fn ident_name(&self) -> Option<(ast::Name, /* is_raw */ bool)> {
- match *self {
- Ident(name, is_raw) => Some((name, is_raw)),
- Interpolated(ref nt) => match **nt {
- NtIdent(ident, is_raw) => Some((ident.name, is_raw)),
- _ => None,
- },
- _ => None,
- }
- }
- /// Returns a lifetime name if this token is a lifetime.
- pub fn lifetime_name(&self) -> Option {
- match *self {
- Lifetime(name) => Some(name),
- Interpolated(ref nt) => match **nt {
- NtLifetime(ident) => Some(ident.name),
- _ => None,
- },
- _ => None,
- }
- }
/// Returns `true` if the token is an identifier.
pub fn is_ident(&self) -> bool {
- self.ident_name().is_some()
+ self.ident().is_some()
}
/// Returns `true` if the token is a lifetime.
crate fn is_lifetime(&self) -> bool {
- self.lifetime_name().is_some()
+ self.lifetime().is_some()
}
/// Returns `true` if the token is a identifier whose name is the given
/// string slice.
crate fn is_ident_named(&self, name: Symbol) -> bool {
- self.ident_name().map_or(false, |(ident_name, _)| ident_name == name)
+ self.ident().map_or(false, |(ident, _)| ident.name == name)
}
/// Returns `true` if the token is an interpolated path.
fn is_path(&self) -> bool {
- if let Interpolated(ref nt) = *self {
+ if let Interpolated(ref nt) = self.kind {
if let NtPath(..) = **nt {
return true;
}
@@ -456,33 +426,27 @@ impl TokenKind {
crate fn is_qpath_start(&self) -> bool {
self == &Lt || self == &BinOp(Shl)
}
-}
-impl Token {
crate fn is_path_start(&self) -> bool {
self == &ModSep || self.is_qpath_start() || self.is_path() ||
self.is_path_segment_keyword() || self.is_ident() && !self.is_reserved_ident()
}
-}
-impl TokenKind {
/// Returns `true` if the token is a given keyword, `kw`.
pub fn is_keyword(&self, kw: Symbol) -> bool {
- self.ident_name().map(|(name, is_raw)| name == kw && !is_raw).unwrap_or(false)
+ self.ident().map(|(id, is_raw)| id.name == kw && !is_raw).unwrap_or(false)
}
- pub fn is_path_segment_keyword(&self) -> bool {
- match self.ident_name() {
- Some((name, false)) => name.is_path_segment_keyword(),
+ crate fn is_path_segment_keyword(&self) -> bool {
+ match self.ident() {
+ Some((id, false)) => id.is_path_segment_keyword(),
_ => false,
}
}
-}
-impl Token {
// Returns true for reserved identifiers used internally for elided lifetimes,
// unnamed method parameters, crate root module, error recovery etc.
- pub fn is_special_ident(&self) -> bool {
+ crate fn is_special_ident(&self) -> bool {
match self.ident() {
Some((id, false)) => id.is_special(),
_ => false,
@@ -512,55 +476,53 @@ impl Token {
_ => false,
}
}
-}
-impl TokenKind {
- crate fn glue(self, joint: TokenKind) -> Option {
- Some(match self {
- Eq => match joint {
+ crate fn glue(self, joint: Token) -> Option {
+ let kind = match self.kind {
+ Eq => match joint.kind {
Eq => EqEq,
Gt => FatArrow,
_ => return None,
},
- Lt => match joint {
+ Lt => match joint.kind {
Eq => Le,
Lt => BinOp(Shl),
Le => BinOpEq(Shl),
BinOp(Minus) => LArrow,
_ => return None,
},
- Gt => match joint {
+ Gt => match joint.kind {
Eq => Ge,
Gt => BinOp(Shr),
Ge => BinOpEq(Shr),
_ => return None,
},
- Not => match joint {
+ Not => match joint.kind {
Eq => Ne,
_ => return None,
},
- BinOp(op) => match joint {
+ BinOp(op) => match joint.kind {
Eq => BinOpEq(op),
BinOp(And) if op == And => AndAnd,
BinOp(Or) if op == Or => OrOr,
Gt if op == Minus => RArrow,
_ => return None,
},
- Dot => match joint {
+ Dot => match joint.kind {
Dot => DotDot,
DotDot => DotDotDot,
_ => return None,
},
- DotDot => match joint {
+ DotDot => match joint.kind {
Dot => DotDotDot,
Eq => DotDotEq,
_ => return None,
},
- Colon => match joint {
+ Colon => match joint.kind {
Colon => ModSep,
_ => return None,
},
- SingleQuote => match joint {
+ SingleQuote => match joint.kind {
Ident(name, false) => Lifetime(Symbol::intern(&format!("'{}", name))),
_ => return None,
},
@@ -570,9 +532,13 @@ impl TokenKind {
Question | OpenDelim(..) | CloseDelim(..) |
Literal(..) | Ident(..) | Lifetime(..) | Interpolated(..) | DocComment(..) |
Whitespace | Comment | Shebang(..) | Eof => return None,
- })
+ };
+
+ Some(Token::new(kind, self.span.to(joint.span)))
}
+}
+impl TokenKind {
/// Returns tokens that are likely to be typed accidentally instead of the current token.
/// Enables better error recovery when the wrong token is found.
crate fn similar_tokens(&self) -> Option> {
@@ -582,14 +548,16 @@ impl TokenKind {
_ => None
}
}
+}
+impl Token {
// See comments in `Nonterminal::to_tokenstream` for why we care about
// *probably* equal here rather than actual equality
crate fn probably_equal_for_proc_macro(&self, other: &TokenKind) -> bool {
- if mem::discriminant(self) != mem::discriminant(other) {
+ if mem::discriminant(&self.kind) != mem::discriminant(other) {
return false
}
- match (self, other) {
+ match (&self.kind, other) {
(&Eq, &Eq) |
(&Lt, &Lt) |
(&Le, &Le) |
@@ -641,9 +609,7 @@ impl TokenKind {
_ => panic!("forgot to add a token?"),
}
}
-}
-impl Token {
crate fn new(kind: TokenKind, span: Span) -> Self {
Token { kind, span }
}
diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs
index 9dea3a4dcc144..d46d2f549c09b 100644
--- a/src/libsyntax/tokenstream.rs
+++ b/src/libsyntax/tokenstream.rs
@@ -430,11 +430,10 @@ impl TokenStreamBuilder {
let last_tree_if_joint = self.0.last().and_then(TokenStream::last_tree_if_joint);
if let Some(TokenTree::Token(last_token)) = last_tree_if_joint {
if let Some((TokenTree::Token(token), is_joint)) = stream.first_tree_and_joint() {
- if let Some(glued_tok) = last_token.kind.glue(token.kind) {
+ if let Some(glued_tok) = last_token.glue(token) {
let last_stream = self.0.pop().unwrap();
self.push_all_but_last_tree(&last_stream);
- let glued_span = last_token.span.to(token.span);
- let glued_tt = TokenTree::token(glued_tok, glued_span);
+ let glued_tt = TokenTree::Token(glued_tok);
let glued_tokenstream = TokenStream::new(vec![(glued_tt, is_joint)]);
self.0.push(glued_tokenstream);
self.push_all_but_first_tree(&stream);
diff --git a/src/libsyntax/util/parser.rs b/src/libsyntax/util/parser.rs
index 9e26f1bf7d374..69dd96625cc02 100644
--- a/src/libsyntax/util/parser.rs
+++ b/src/libsyntax/util/parser.rs
@@ -1,4 +1,4 @@
-use crate::parse::token::{self, TokenKind, BinOpToken};
+use crate::parse::token::{self, Token, BinOpToken};
use crate::symbol::kw;
use crate::ast::{self, BinOpKind};
@@ -69,9 +69,9 @@ pub enum Fixity {
impl AssocOp {
/// Creates a new AssocOP from a token
- pub fn from_token(t: &TokenKind) -> Option {
+ pub fn from_token(t: &Token) -> Option {
use AssocOp::*;
- match *t {
+ match t.kind {
token::BinOpEq(k) => Some(AssignOp(k)),
token::Eq => Some(Assign),
token::BinOp(BinOpToken::Star) => Some(Multiply),