Skip to content

Commit

Permalink
Simplify conversions between tokens and semantic literals
Browse files Browse the repository at this point in the history
  • Loading branch information
@petrochenkov
petrochenkov committed May 11, 2019
1 parent a5b3f33 commit 8739668
Show file tree
Hide file tree
Showing 10 changed files with 259 additions and 324 deletions.
56 changes: 3 additions & 53 deletions src/librustc/hir/print.rs
View file
Expand Up @@ -5,7 +5,7 @@ use syntax::parse::ParseSess;
use syntax::parse::lexer::comments; use syntax::parse::lexer::comments;
use syntax::print::pp::{self, Breaks}; use syntax::print::pp::{self, Breaks};
use syntax::print::pp::Breaks::{Consistent, Inconsistent}; use syntax::print::pp::Breaks::{Consistent, Inconsistent};
use syntax::print::pprust::PrintState; use syntax::print::pprust::{self, PrintState};
use syntax::ptr::P; use syntax::ptr::P;
use syntax::symbol::keywords; use syntax::symbol::keywords;
use syntax::util::parser::{self, AssocOp, Fixity}; use syntax::util::parser::{self, AssocOp, Fixity};
Expand All @@ -15,7 +15,6 @@ use crate::hir;
use crate::hir::{PatKind, GenericBound, TraitBoundModifier, RangeEnd}; use crate::hir::{PatKind, GenericBound, TraitBoundModifier, RangeEnd};
use crate::hir::{GenericParam, GenericParamKind, GenericArg}; use crate::hir::{GenericParam, GenericParamKind, GenericArg};


use std::ascii;
use std::borrow::Cow; use std::borrow::Cow;
use std::cell::Cell; use std::cell::Cell;
use std::io::{self, Write, Read}; use std::io::{self, Write, Read};
Expand Down Expand Up @@ -1251,57 +1250,8 @@ impl<'a> State<'a> {


fn print_literal(&mut self, lit: &hir::Lit) -> io::Result<()> { fn print_literal(&mut self, lit: &hir::Lit) -> io::Result<()> {
self.maybe_print_comment(lit.span.lo())?; self.maybe_print_comment(lit.span.lo())?;
match lit.node { let (token, suffix) = lit.node.to_lit_token();
hir::LitKind::Str(st, style) => self.print_string(&st.as_str(), style), self.writer().word(pprust::literal_to_string(token, suffix))
hir::LitKind::Err(st) => {
let st = st.as_str().escape_debug().to_string();
let mut res = String::with_capacity(st.len() + 2);
res.push('\'');
res.push_str(&st);
res.push('\'');
self.writer().word(res)
}
hir::LitKind::Byte(byte) => {
let mut res = String::from("b'");
res.extend(ascii::escape_default(byte).map(|c| c as char));
res.push('\'');
self.writer().word(res)
}
hir::LitKind::Char(ch) => {
let mut res = String::from("'");
res.extend(ch.escape_default());
res.push('\'');
self.writer().word(res)
}
hir::LitKind::Int(i, t) => {
match t {
ast::LitIntType::Signed(st) => {
self.writer().word(st.val_to_string(i as i128))
}
ast::LitIntType::Unsigned(ut) => {
self.writer().word(ut.val_to_string(i))
}
ast::LitIntType::Unsuffixed => {
self.writer().word(i.to_string())
}
}
}
hir::LitKind::Float(ref f, t) => {
self.writer().word(format!("{}{}", &f, t.ty_to_string()))
}
hir::LitKind::FloatUnsuffixed(ref f) => self.writer().word(f.as_str().to_string()),
hir::LitKind::Bool(val) => {
if val { self.writer().word("true") } else { self.writer().word("false") }
}
hir::LitKind::ByteStr(ref v) => {
let mut escaped: String = String::new();
for &ch in v.iter() {
escaped.extend(ascii::escape_default(ch)
.map(|c| c as char));
}
self.writer().word(format!("b\"{}\"", escaped))
}
}
} }


pub fn print_expr(&mut self, expr: &hir::Expr) -> io::Result<()> { pub fn print_expr(&mut self, expr: &hir::Expr) -> io::Result<()> {
Expand Down
24 changes: 10 additions & 14 deletions src/librustdoc/clean/cfg.rs
View file
Expand Up @@ -591,13 +591,10 @@ mod test {
let mi = dummy_meta_item_word("all"); let mi = dummy_meta_item_word("all");
assert_eq!(Cfg::parse(&mi), Ok(word_cfg("all"))); assert_eq!(Cfg::parse(&mi), Ok(word_cfg("all")));


let node = LitKind::Str(Symbol::intern("done"), StrStyle::Cooked); let mi = attr::mk_name_value_item_str(
let (token, suffix) = node.lit_token(); Ident::from_str("all"),
let mi = MetaItem { dummy_spanned(Symbol::intern("done"))
path: Path::from_ident(Ident::from_str("all")), );
node: MetaItemKind::NameValue(Lit { node, token, suffix, span: DUMMY_SP }),
span: DUMMY_SP,
};
assert_eq!(Cfg::parse(&mi), Ok(name_value_cfg("all", "done"))); assert_eq!(Cfg::parse(&mi), Ok(name_value_cfg("all", "done")));


let mi = dummy_meta_item_list!(all, [a, b]); let mi = dummy_meta_item_list!(all, [a, b]);
Expand Down Expand Up @@ -625,13 +622,12 @@ mod test {
#[test] #[test]
fn test_parse_err() { fn test_parse_err() {
with_globals(|| { with_globals(|| {
let node = LitKind::Bool(false); let mi = attr::mk_name_value_item(
let (token, suffix) = node.lit_token(); DUMMY_SP,
let mi = MetaItem { Ident::from_str("foo"),
path: Path::from_ident(Ident::from_str("foo")), LitKind::Bool(false),
node: MetaItemKind::NameValue(Lit { node, token, suffix, span: DUMMY_SP }), DUMMY_SP,
span: DUMMY_SP, );
};
assert!(Cfg::parse(&mi).is_err()); assert!(Cfg::parse(&mi).is_err());


let mi = dummy_meta_item_list!(not, [a, b]); let mi = dummy_meta_item_list!(not, [a, b]);
Expand Down
104 changes: 58 additions & 46 deletions src/libsyntax/attr/mod.rs
View file
Expand Up @@ -14,7 +14,7 @@ pub use StabilityLevel::*;
use crate::ast; use crate::ast;
use crate::ast::{AttrId, Attribute, AttrStyle, Name, Ident, Path, PathSegment}; use crate::ast::{AttrId, Attribute, AttrStyle, Name, Ident, Path, PathSegment};
use crate::ast::{MetaItem, MetaItemKind, NestedMetaItem}; use crate::ast::{MetaItem, MetaItemKind, NestedMetaItem};
use crate::ast::{Lit, LitKind, Expr, ExprKind, Item, Local, Stmt, StmtKind, GenericParam}; use crate::ast::{Lit, LitKind, Expr, Item, Local, Stmt, StmtKind, GenericParam};
use crate::mut_visit::visit_clobber; use crate::mut_visit::visit_clobber;
use crate::source_map::{BytePos, Spanned, dummy_spanned}; use crate::source_map::{BytePos, Spanned, dummy_spanned};
use crate::parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration}; use crate::parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration};
Expand All @@ -27,9 +27,11 @@ use crate::ThinVec;
use crate::tokenstream::{TokenStream, TokenTree, DelimSpan}; use crate::tokenstream::{TokenStream, TokenTree, DelimSpan};
use crate::GLOBALS; use crate::GLOBALS;


use errors::Handler;
use log::debug; use log::debug;
use syntax_pos::{FileName, Span}; use syntax_pos::{FileName, Span};


use std::ascii;
use std::iter; use std::iter;
use std::ops::DerefMut; use std::ops::DerefMut;


Expand Down Expand Up @@ -350,14 +352,13 @@ impl Attribute {
/* Constructors */ /* Constructors */


pub fn mk_name_value_item_str(ident: Ident, value: Spanned<Symbol>) -> MetaItem { pub fn mk_name_value_item_str(ident: Ident, value: Spanned<Symbol>) -> MetaItem {
let node = LitKind::Str(value.node, ast::StrStyle::Cooked); let lit_kind = LitKind::Str(value.node, ast::StrStyle::Cooked);
let (token, suffix) = node.lit_token(); mk_name_value_item(ident.span.to(value.span), ident, lit_kind, value.span)
let value = Lit { node, token, suffix, span: value.span };
mk_name_value_item(ident.span.to(value.span), ident, value)
} }


pub fn mk_name_value_item(span: Span, ident: Ident, value: Lit) -> MetaItem { pub fn mk_name_value_item(span: Span, ident: Ident, lit_kind: LitKind, lit_span: Span) -> MetaItem {
MetaItem { path: Path::from_ident(ident), span, node: MetaItemKind::NameValue(value) } let lit = Lit::from_lit_kind(lit_kind, lit_span);
MetaItem { path: Path::from_ident(ident), span, node: MetaItemKind::NameValue(lit) }
} }


pub fn mk_list_item(span: Span, ident: Ident, items: Vec<NestedMetaItem>) -> MetaItem { pub fn mk_list_item(span: Span, ident: Ident, items: Vec<NestedMetaItem>) -> MetaItem {
Expand Down Expand Up @@ -419,9 +420,8 @@ pub fn mk_spanned_attr_outer(sp: Span, id: AttrId, item: MetaItem) -> Attribute


pub fn mk_sugared_doc_attr(id: AttrId, text: Symbol, span: Span) -> Attribute { pub fn mk_sugared_doc_attr(id: AttrId, text: Symbol, span: Span) -> Attribute {
let style = doc_comment_style(&text.as_str()); let style = doc_comment_style(&text.as_str());
let node = LitKind::Str(text, ast::StrStyle::Cooked); let lit_kind = LitKind::Str(text, ast::StrStyle::Cooked);
let (token, suffix) = node.lit_token(); let lit = Lit::from_lit_kind(lit_kind, span);
let lit = Lit { node, token, suffix, span };
Attribute { Attribute {
id, id,
style, style,
Expand Down Expand Up @@ -565,9 +565,7 @@ impl MetaItemKind {
Some(TokenTree::Token(_, token::Eq)) => { Some(TokenTree::Token(_, token::Eq)) => {
tokens.next(); tokens.next();
return if let Some(TokenTree::Token(span, token)) = tokens.next() { return if let Some(TokenTree::Token(span, token)) = tokens.next() {
LitKind::from_token(token).map(|(node, token, suffix)| { Lit::from_token(&token, span, None).map(MetaItemKind::NameValue)
MetaItemKind::NameValue(Lit { node, token, suffix, span })
})
} else { } else {
None None
}; };
Expand Down Expand Up @@ -612,9 +610,9 @@ impl NestedMetaItem {
where I: Iterator<Item = TokenTree>, where I: Iterator<Item = TokenTree>,
{ {
if let Some(TokenTree::Token(span, token)) = tokens.peek().cloned() { if let Some(TokenTree::Token(span, token)) = tokens.peek().cloned() {
if let Some((node, token, suffix)) = LitKind::from_token(token) { if let Some(lit) = Lit::from_token(&token, span, None) {
tokens.next(); tokens.next();
return Some(NestedMetaItem::Literal(Lit { node, token, suffix, span })); return Some(NestedMetaItem::Literal(lit));
} }
} }


Expand All @@ -624,21 +622,19 @@ impl NestedMetaItem {


impl Lit { impl Lit {
crate fn tokens(&self) -> TokenStream { crate fn tokens(&self) -> TokenStream {
TokenTree::Token(self.span, self.node.token()).into() let token = match self.token {
token::Bool(symbol) => Token::Ident(Ident::with_empty_ctxt(symbol), false),
token => Token::Literal(token, self.suffix),
};
TokenTree::Token(self.span, token).into()
} }
} }


impl LitKind { impl LitKind {
fn token(&self) -> Token { /// Attempts to recover a token from semantic literal.
match self.lit_token() { /// This function is used when the original token doesn't exist (e.g. the literal is created
(token::Bool(symbol), _) => Token::Ident(Ident::with_empty_ctxt(symbol), false), /// by an AST-based macro) or unavailable (e.g. from HIR pretty-printing).
(lit, suffix) => Token::Literal(lit, suffix), pub fn to_lit_token(&self) -> (token::Lit, Option<Symbol>) {
}
}

pub fn lit_token(&self) -> (token::Lit, Option<Symbol>) {
use std::ascii;

match *self { match *self {
LitKind::Str(string, ast::StrStyle::Cooked) => { LitKind::Str(string, ast::StrStyle::Cooked) => {
let escaped = string.as_str().escape_default().to_string(); let escaped = string.as_str().escape_default().to_string();
Expand Down Expand Up @@ -679,29 +675,45 @@ impl LitKind {
LitKind::Err(val) => (token::Lit::Err(val), None), LitKind::Err(val) => (token::Lit::Err(val), None),
} }
} }
}


fn from_token(token: Token) -> Option<(LitKind, token::Lit, Option<Symbol>)> { impl Lit {
match token { /// Converts literal token with a suffix into an AST literal.
Token::Ident(ident, false) if ident.name == keywords::True.name() => /// Works speculatively and may return `None` is diagnostic handler is not passed.
Some((LitKind::Bool(true), token::Bool(ident.name), None)), /// If diagnostic handler is passed, may return `Some`,
Token::Ident(ident, false) if ident.name == keywords::False.name() => /// possibly after reporting non-fatal errors and recovery, or `None` for irrecoverable errors.
Some((LitKind::Bool(false), token::Bool(ident.name), None)), crate fn from_token(
Token::Interpolated(nt) => match *nt { token: &token::Token,
token::NtExpr(ref v) | token::NtLiteral(ref v) => match v.node { span: Span,
ExprKind::Lit(ref lit) => Some((lit.node.clone(), lit.token, lit.suffix)), diag: Option<(Span, &Handler)>,
_ => None, ) -> Option<Lit> {
}, let (token, suffix) = match *token {
_ => None, token::Ident(ident, false) if ident.name == keywords::True.name() ||
}, ident.name == keywords::False.name() =>
Token::Literal(lit, suf) => { (token::Bool(ident.name), None),
let (suffix_illegal, result) = parse::lit_token(lit, suf, None); token::Literal(token, suffix) =>
if result.is_none() || suffix_illegal && suf.is_some() { (token, suffix),
return None; token::Interpolated(ref nt) => {
if let token::NtExpr(expr) | token::NtLiteral(expr) = &**nt {
if let ast::ExprKind::Lit(lit) = &expr.node {
return Some(lit.clone());
}
} }
Some((result.unwrap(), lit, suf)) return None;
} }
_ => None, _ => return None,
} };

let node = LitKind::from_lit_token(token, suffix, diag)?;
Some(Lit { node, token, suffix, span })
}

/// Attempts to recover an AST literal from semantic literal.
/// This function is used when the original token doesn't exist (e.g. the literal is created
/// by an AST-based macro) or unavailable (e.g. from HIR pretty-printing).
pub fn from_lit_kind(node: LitKind, span: Span) -> Lit {
let (token, suffix) = node.to_lit_token();
Lit { node, token, suffix, span }
} }
} }


Expand Down
11 changes: 5 additions & 6 deletions src/libsyntax/ext/build.rs
View file
Expand Up @@ -697,9 +697,9 @@ impl<'a> AstBuilder for ExtCtxt<'a> {
self.expr_struct(span, self.path_ident(span, id), fields) self.expr_struct(span, self.path_ident(span, id), fields)
} }


fn expr_lit(&self, span: Span, node: ast::LitKind) -> P<ast::Expr> { fn expr_lit(&self, span: Span, lit_kind: ast::LitKind) -> P<ast::Expr> {
let (token, suffix) = node.lit_token(); let lit = ast::Lit::from_lit_kind(lit_kind, span);
self.expr(span, ast::ExprKind::Lit(ast::Lit { node, token, suffix, span })) self.expr(span, ast::ExprKind::Lit(lit))
} }
fn expr_usize(&self, span: Span, i: usize) -> P<ast::Expr> { fn expr_usize(&self, span: Span, i: usize) -> P<ast::Expr> {
self.expr_lit(span, ast::LitKind::Int(i as u128, self.expr_lit(span, ast::LitKind::Int(i as u128,
Expand Down Expand Up @@ -1165,11 +1165,10 @@ impl<'a> AstBuilder for ExtCtxt<'a> {
attr::mk_list_item(sp, Ident::with_empty_ctxt(name).with_span_pos(sp), mis) attr::mk_list_item(sp, Ident::with_empty_ctxt(name).with_span_pos(sp), mis)
} }


fn meta_name_value(&self, span: Span, name: ast::Name, node: ast::LitKind) fn meta_name_value(&self, span: Span, name: ast::Name, lit_kind: ast::LitKind)
-> ast::MetaItem { -> ast::MetaItem {
let (token, suffix) = node.lit_token();
attr::mk_name_value_item(span, Ident::with_empty_ctxt(name).with_span_pos(span), attr::mk_name_value_item(span, Ident::with_empty_ctxt(name).with_span_pos(span),
ast::Lit { node, token, suffix, span }) lit_kind, span)
} }


fn item_use(&self, sp: Span, fn item_use(&self, sp: Span,
Expand Down

0 comments on commit 8739668

Please sign in to comment.