Skip to content
Permalink
Browse files

Move literal parsing code into a separate file

Remove some dead code
  • Loading branch information...
petrochenkov committed May 10, 2019
1 parent 8739668 commit 3f064cae3d9d0d33951a44c30d83696563244572
@@ -27,11 +27,9 @@ use crate::ThinVec;
use crate::tokenstream::{TokenStream, TokenTree, DelimSpan};
use crate::GLOBALS;

use errors::Handler;
use log::debug;
use syntax_pos::{FileName, Span};

use std::ascii;
use std::iter;
use std::ops::DerefMut;

@@ -620,103 +618,6 @@ impl NestedMetaItem {
}
}

impl Lit {
crate fn tokens(&self) -> TokenStream {
let token = match self.token {
token::Bool(symbol) => Token::Ident(Ident::with_empty_ctxt(symbol), false),
token => Token::Literal(token, self.suffix),
};
TokenTree::Token(self.span, token).into()
}
}

impl LitKind {
/// Attempts to recover a token from semantic literal.
/// This function is used when the original token doesn't exist (e.g. the literal is created
/// by an AST-based macro) or unavailable (e.g. from HIR pretty-printing).
pub fn to_lit_token(&self) -> (token::Lit, Option<Symbol>) {
match *self {
LitKind::Str(string, ast::StrStyle::Cooked) => {
let escaped = string.as_str().escape_default().to_string();
(token::Lit::Str_(Symbol::intern(&escaped)), None)
}
LitKind::Str(string, ast::StrStyle::Raw(n)) => {
(token::Lit::StrRaw(string, n), None)
}
LitKind::ByteStr(ref bytes) => {
let string = bytes.iter().cloned().flat_map(ascii::escape_default)
.map(Into::<char>::into).collect::<String>();
(token::Lit::ByteStr(Symbol::intern(&string)), None)
}
LitKind::Byte(byte) => {
let string: String = ascii::escape_default(byte).map(Into::<char>::into).collect();
(token::Lit::Byte(Symbol::intern(&string)), None)
}
LitKind::Char(ch) => {
let string: String = ch.escape_default().map(Into::<char>::into).collect();
(token::Lit::Char(Symbol::intern(&string)), None)
}
LitKind::Int(n, ty) => {
let suffix = match ty {
ast::LitIntType::Unsigned(ty) => Some(Symbol::intern(ty.ty_to_string())),
ast::LitIntType::Signed(ty) => Some(Symbol::intern(ty.ty_to_string())),
ast::LitIntType::Unsuffixed => None,
};
(token::Lit::Integer(Symbol::intern(&n.to_string())), suffix)
}
LitKind::Float(symbol, ty) => {
(token::Lit::Float(symbol), Some(Symbol::intern(ty.ty_to_string())))
}
LitKind::FloatUnsuffixed(symbol) => (token::Lit::Float(symbol), None),
LitKind::Bool(value) => {
let kw = if value { keywords::True } else { keywords::False };
(token::Lit::Bool(kw.name()), None)
}
LitKind::Err(val) => (token::Lit::Err(val), None),
}
}
}

impl Lit {
/// Converts literal token with a suffix into an AST literal.
/// Works speculatively and may return `None` is diagnostic handler is not passed.
/// If diagnostic handler is passed, may return `Some`,
/// possibly after reporting non-fatal errors and recovery, or `None` for irrecoverable errors.
crate fn from_token(
token: &token::Token,
span: Span,
diag: Option<(Span, &Handler)>,
) -> Option<Lit> {
let (token, suffix) = match *token {
token::Ident(ident, false) if ident.name == keywords::True.name() ||
ident.name == keywords::False.name() =>
(token::Bool(ident.name), None),
token::Literal(token, suffix) =>
(token, suffix),
token::Interpolated(ref nt) => {
if let token::NtExpr(expr) | token::NtLiteral(expr) = &**nt {
if let ast::ExprKind::Lit(lit) = &expr.node {
return Some(lit.clone());
}
}
return None;
}
_ => return None,
};

let node = LitKind::from_lit_token(token, suffix, diag)?;
Some(Lit { node, token, suffix, span })
}

/// Attempts to recover an AST literal from semantic literal.
/// This function is used when the original token doesn't exist (e.g. the literal is created
/// by an AST-based macro) or unavailable (e.g. from HIR pretty-printing).
pub fn from_lit_kind(node: LitKind, span: Span) -> Lit {
let (token, suffix) = node.to_lit_token();
Lit { node, token, suffix, span }
}
}

pub trait HasAttrs: Sized {
fn attrs(&self) -> &[ast::Attribute];
fn visit_attrs<F: FnOnce(&mut Vec<ast::Attribute>)>(&mut self, f: F);
@@ -25,16 +25,3 @@ pub fn expr_requires_semi_to_be_stmt(e: &ast::Expr) -> bool {
_ => true,
}
}

/// this statement requires a semicolon after it.
/// note that in one case (`stmt_semi`), we've already
/// seen the semicolon, and thus don't need another.
pub fn stmt_ends_with_semi(stmt: &ast::StmtKind) -> bool {
match *stmt {
ast::StmtKind::Local(_) => true,
ast::StmtKind::Expr(ref e) => expr_requires_semi_to_be_stmt(e),
ast::StmtKind::Item(_) |
ast::StmtKind::Semi(..) |
ast::StmtKind::Mac(..) => false,
}
}
@@ -262,18 +262,6 @@ impl<'a> StringReader<'a> {
}
}

pub fn new(sess: &'a ParseSess,
source_file: Lrc<syntax_pos::SourceFile>,
override_span: Option<Span>) -> Self {
let mut sr = StringReader::new_raw(sess, source_file, override_span);
if sr.advance_token().is_err() {
sr.emit_fatal_errors();
FatalError.raise();
}

sr
}

pub fn new_or_buffered_errs(sess: &'a ParseSess,
source_file: Lrc<syntax_pos::SourceFile>,
override_span: Option<Span>) -> Result<Self, Vec<Diagnostic>> {
@@ -1627,7 +1615,12 @@ mod tests {
teststr: String)
-> StringReader<'a> {
let sf = sm.new_source_file(PathBuf::from(teststr.clone()).into(), teststr);
StringReader::new(sess, sf, None)
let mut sr = StringReader::new_raw(sess, sf, None);
if sr.advance_token().is_err() {
sr.emit_fatal_errors();
FatalError.raise();
}
sr
}

#[test]
Oops, something went wrong.

0 comments on commit 3f064ca

Please sign in to comment.
You can’t perform that action at this time.