Skip to content
Permalink
Browse files

syntax: Move most of the `TokenKind` methods to `Token`

  • Loading branch information...
petrochenkov committed Jun 8, 2019
1 parent ffe2347 commit 0ca3c2f881fc4bc51bfa92f1adcd1b845b812534
@@ -257,7 +257,7 @@ impl<'a> Classifier<'a> {
token::Question => Class::QuestionMark,

token::Dollar => {
if self.lexer.peek().kind.is_ident() {
if self.lexer.peek().is_ident() {
self.in_macro_nonterminal = true;
Class::MacroNonTerminal
} else {
@@ -20,7 +20,7 @@ use crate::source_map::{BytePos, Spanned, dummy_spanned};
use crate::parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration};
use crate::parse::parser::Parser;
use crate::parse::{self, ParseSess, PResult};
use crate::parse::token::{self, Token, TokenKind};
use crate::parse::token::{self, Token};
use crate::ptr::P;
use crate::symbol::{sym, Symbol};
use crate::ThinVec;
@@ -467,8 +467,7 @@ impl MetaItem {
segment.ident.span.ctxt());
idents.push(TokenTree::token(token::ModSep, mod_sep_span).into());
}
idents.push(TokenTree::token(TokenKind::from_ast_ident(segment.ident),
segment.ident.span).into());
idents.push(TokenTree::Token(Token::from_ast_ident(segment.ident)).into());
last_pos = segment.ident.span.hi();
}
self.node.tokens(self.span).append_to_tree_and_joint_vec(&mut idents);
@@ -428,13 +428,13 @@ pub fn parse_failure_msg(tok: TokenKind) -> String {
}

/// Performs a token equality check, ignoring syntax context (that is, an unhygienic comparison)
fn token_name_eq(t1: &TokenKind, t2: &TokenKind) -> bool {
if let (Some((name1, is_raw1)), Some((name2, is_raw2))) = (t1.ident_name(), t2.ident_name()) {
name1 == name2 && is_raw1 == is_raw2
} else if let (Some(name1), Some(name2)) = (t1.lifetime_name(), t2.lifetime_name()) {
name1 == name2
fn token_name_eq(t1: &Token, t2: &Token) -> bool {
if let (Some((ident1, is_raw1)), Some((ident2, is_raw2))) = (t1.ident(), t2.ident()) {
ident1.name == ident2.name && is_raw1 == is_raw2
} else if let (Some(ident1), Some(ident2)) = (t1.lifetime(), t2.lifetime()) {
ident1.name == ident2.name
} else {
*t1 == *t2
t1.kind == t2.kind
}
}

@@ -712,7 +712,7 @@ pub fn parse(

// If we reached the EOF, check that there is EXACTLY ONE possible matcher. Otherwise,
// either the parse is ambiguous (which should never happen) or there is a syntax error.
if token_name_eq(&parser.token, &token::Eof) {
if parser.token == token::Eof {
if eof_items.len() == 1 {
let matches = eof_items[0]
.matches
@@ -4,7 +4,7 @@ use crate::ext::expand::Marker;
use crate::ext::tt::macro_parser::{MatchedNonterminal, MatchedSeq, NamedMatch};
use crate::ext::tt::quoted;
use crate::mut_visit::noop_visit_tt;
use crate::parse::token::{self, NtTT, Token, TokenKind};
use crate::parse::token::{self, NtTT, Token};
use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint};

use smallvec::{smallvec, SmallVec};
@@ -237,7 +237,7 @@ pub fn transcribe(
Ident::new(ident.name, ident.span.apply_mark(cx.current_expansion.mark));
sp = sp.apply_mark(cx.current_expansion.mark);
result.push(TokenTree::token(token::Dollar, sp).into());
result.push(TokenTree::token(TokenKind::from_ast_ident(ident), sp).into());
result.push(TokenTree::Token(Token::from_ast_ident(ident)).into());
}
}

@@ -1501,7 +1501,7 @@ fn char_at(s: &str, byte: usize) -> char {
mod tests {
use super::*;

use crate::ast::{Ident, CrateConfig};
use crate::ast::CrateConfig;
use crate::symbol::Symbol;
use crate::source_map::{SourceMap, FilePathMapping};
use crate::feature_gate::UnstableFeatures;
@@ -1562,7 +1562,7 @@ mod tests {
assert_eq!(string_reader.next_token(), token::Whitespace);
let tok1 = string_reader.next_token();
let tok2 = Token::new(
token::Ident(Symbol::intern("fn"), false),
mk_ident("fn"),
Span::new(BytePos(21), BytePos(23), NO_EXPANSION),
);
assert_eq!(tok1.kind, tok2.kind);
@@ -1593,7 +1593,7 @@ mod tests {

// make the identifier by looking up the string in the interner
fn mk_ident(id: &str) -> TokenKind {
TokenKind::from_ast_ident(Ident::from_str(id))
token::Ident(Symbol::intern(id), false)
}

fn mk_lit(kind: token::LitKind, symbol: &str, suffix: Option<&str>) -> TokenKind {
@@ -2627,9 +2627,11 @@ impl<'a> Parser<'a> {
token::Ident(name, _) => name,
_ => unreachable!()
};
let mut err = self.fatal(&format!("unknown macro variable `{}`", name));
err.span_label(self.token.span, "unknown macro variable");
err.emit();
let span = self.prev_span.to(self.token.span);
self.diagnostic()
.struct_span_fatal(span, &format!("unknown macro variable `{}`", name))
.span_label(span, "unknown macro variable")
.emit();
self.bump();
return
}

0 comments on commit 0ca3c2f

Please sign in to comment.
You can’t perform that action at this time.