Skip to content

Commit

Permalink
TokenTree: Op -> Punct, Term -> Ident
Browse files Browse the repository at this point in the history
  • Loading branch information
@petrochenkov
petrochenkov committed May 15, 2018
1 parent decc619 commit 47d4089
Show file tree
Hide file tree
Showing 7 changed files with 115 additions and 131 deletions.
156 changes: 70 additions & 86 deletions src/libproc_macro/lib.rs
View file
Expand Up @@ -246,7 +246,7 @@ pub mod token_stream {

/// `quote!(..)` accepts arbitrary tokens and expands into a `TokenStream` describing the input.
/// For example, `quote!(a + b)` will produce a expression, that, when evaluated, constructs
/// the `TokenStream` `[Word("a"), Op('+', Alone), Word("b")]`.
/// the `TokenStream` `[Word("a"), Punct('+', Alone), Word("b")]`.
///
/// Unquoting is done with `$`, and works by taking the single next ident as the unquoted term.
/// To quote `$` itself, use `$$`.
Expand Down Expand Up @@ -499,17 +499,9 @@ pub enum TokenTree {
/// A token stream surrounded by bracket delimiters.
Group(Group),
/// An identifier or lifetime identifier.
///
/// REVIEW Maybe let's name it `Ident` instead of inventing a new term, it's named "identifier"
/// REVIEW everywhere in the compiler, including `ident` in `macro`/`macro_rules!` DSL.
Term(Term),
Ident(Ident),
/// A single punctuation character (`+`, `,`, `$`, etc.).
///
/// REVIEW This is not an operator, operators are more narrow set, they also can be
/// REVIEW multicharacter, this is punctuation, even the comment says so!
/// REVIEW @dtolnay suggested `Punct` in the original implementation PR too, and it was
/// REVIEW received positively, but the renaming never actually happened.
Op(Op),
Punct(Punct),
/// A literal character (`'a'`), string (`"hello"`), number (`2.3`), etc.
Literal(Literal),
}
Expand All @@ -526,8 +518,8 @@ impl TokenTree {
pub fn span(&self) -> Span {
match *self {
TokenTree::Group(ref t) => t.span(),
TokenTree::Term(ref t) => t.span(),
TokenTree::Op(ref t) => t.span(),
TokenTree::Ident(ref t) => t.span(),
TokenTree::Punct(ref t) => t.span(),
TokenTree::Literal(ref t) => t.span(),
}
}
Expand All @@ -541,8 +533,8 @@ impl TokenTree {
pub fn set_span(&mut self, span: Span) {
match *self {
TokenTree::Group(ref mut t) => t.set_span(span),
TokenTree::Term(ref mut t) => t.set_span(span),
TokenTree::Op(ref mut t) => t.set_span(span),
TokenTree::Ident(ref mut t) => t.set_span(span),
TokenTree::Punct(ref mut t) => t.set_span(span),
TokenTree::Literal(ref mut t) => t.set_span(span),
}
}
Expand All @@ -556,16 +548,16 @@ impl fmt::Debug for TokenTree {
// so don't bother with an extra layer of indirection
match *self {
TokenTree::Group(ref tt) => tt.fmt(f),
TokenTree::Term(ref tt) => tt.fmt(f),
TokenTree::Op(ref tt) => tt.fmt(f),
TokenTree::Ident(ref tt) => tt.fmt(f),
TokenTree::Punct(ref tt) => tt.fmt(f),
TokenTree::Literal(ref tt) => tt.fmt(f),
}
}
}

/// REVIEW the impls below are kind of `From<T> for Option<T>`, not strictly necessary,
/// REVIEW but convenient. No harm, I guess. I'd actually like to see impls
/// REVIEW `From<Group/Term/Op/Literal> for TokenStream` to avoid stuttering like
/// REVIEW `From<Group/Ident/Punct/Literal> for TokenStream` to avoid stuttering like
/// REVIEW `TokenTree::Literal(Literal::string("lalala")).into()`.
#[unstable(feature = "proc_macro", issue = "38356")]
impl From<Group> for TokenTree {
Expand All @@ -575,16 +567,16 @@ impl From<Group> for TokenTree {
}

#[unstable(feature = "proc_macro", issue = "38356")]
impl From<Term> for TokenTree {
fn from(g: Term) -> TokenTree {
TokenTree::Term(g)
impl From<Ident> for TokenTree {
fn from(g: Ident) -> TokenTree {
TokenTree::Ident(g)
}
}

#[unstable(feature = "proc_macro", issue = "38356")]
impl From<Op> for TokenTree {
fn from(g: Op) -> TokenTree {
TokenTree::Op(g)
impl From<Punct> for TokenTree {
fn from(g: Punct) -> TokenTree {
TokenTree::Punct(g)
}
}

Expand All @@ -603,8 +595,8 @@ impl fmt::Display for TokenTree {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match *self {
TokenTree::Group(ref t) => t.fmt(f),
TokenTree::Term(ref t) => t.fmt(f),
TokenTree::Op(ref t) => t.fmt(f),
TokenTree::Ident(ref t) => t.fmt(f),
TokenTree::Punct(ref t) => t.fmt(f),
TokenTree::Literal(ref t) => t.fmt(f),
}
}
Expand Down Expand Up @@ -703,37 +695,32 @@ impl fmt::Display for Group {
}
}

/// An `Op` is an single punctuation character like `+`, `-` or `#`.
/// An `Punct` is an single punctuation character like `+`, `-` or `#`.
///
/// Multicharacter operators like `+=` are represented as two instances of `Op` with different
/// Multicharacter operators like `+=` are represented as two instances of `Punct` with different
/// forms of `Spacing` returned.
///
/// REVIEW This is not an operator, operators are more narrow set, they also can be
/// REVIEW multicharacter, this is punctuation, even the comment says so!
/// REVIEW @dtolnay suggested `Punct` in the original implementation PR too, and it was
/// REVIEW received positively, but the renaming never actually happened.
///
/// REVIEW We should guarantee that `Op` contains a valid punctuation character permitted by
/// REVIEW We should guarantee that `Punct` contains a valid punctuation character permitted by
/// REVIEW the language and not a random unicode code point. The check is already performed in
/// REVIEW `TokenTree::to_internal`, but we should do it on construction.
/// REVIEW `Op` can also avoid using `char` internally and keep an u8-like enum.
/// REVIEW `Punct` can also avoid using `char` internally and keep an u8-like enum.
///
/// REVIEW ATTENTION: `Copy` impl on a struct with private fields.
/// REVIEW Do we want to guarantee `Op` to be `Copy`?
/// REVIEW Do we want to guarantee `Punct` to be `Copy`?
#[unstable(feature = "proc_macro", issue = "38356")]
#[derive(Copy, Clone, Debug)]
pub struct Op {
op: char,
pub struct Punct {
ch: char,
spacing: Spacing,
span: Span,
}

#[unstable(feature = "proc_macro", issue = "38356")]
impl !Send for Op {}
impl !Send for Punct {}
#[unstable(feature = "proc_macro", issue = "38356")]
impl !Sync for Op {}
impl !Sync for Punct {}

/// Whether an `Op` is followed immediately by another `Op` or
/// Whether an `Punct` is followed immediately by another `Punct` or
/// followed by another token or whitespace.
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
#[unstable(feature = "proc_macro", issue = "38356")]
Expand All @@ -744,10 +731,10 @@ pub enum Spacing {
Joint,
}

impl Op {
/// Creates a new `Op` from the given character and spacing.
impl Punct {
/// Creates a new `Punct` from the given character and spacing.
///
/// The returned `Op` will have the default span of `Span::call_site()`
/// The returned `Punct` will have the default span of `Span::call_site()`
/// which can be further configured with the `set_span` method below.
///
/// REVIEW Why we even use `char` here? There's no reason to use unicode here.
Expand All @@ -756,9 +743,9 @@ impl Op {
/// REVIEW TO_DO Do input validation on construction, the argument should be a valid punctuation
/// REVIEW character permitted by the language.
#[unstable(feature = "proc_macro", issue = "38356")]
pub fn new(op: char, spacing: Spacing) -> Op {
Op {
op: op,
pub fn new(ch: char, spacing: Spacing) -> Punct {
Punct {
ch: ch,
spacing: spacing,
span: Span::call_site(),
}
Expand All @@ -770,12 +757,12 @@ impl Op {
/// REVIEW except for maybe future compatibility in case Rust turns into APL,
/// REVIEW but if it's more convenient to use `char` then that's okay.
#[unstable(feature = "proc_macro", issue = "38356")]
pub fn op(&self) -> char {
self.op
pub fn as_char(&self) -> char {
self.ch
}

/// Returns the spacing of this punctuation character, indicating whether it's immediately
/// followed by another `Op` in the token stream, so they can potentially be combined into
/// followed by another `Punct` in the token stream, so they can potentially be combined into
/// a multicharacter operator (`Joint`), or it's followed by some other token or whitespace
/// (`Alone`) so the operator has certainly ended.
#[unstable(feature = "proc_macro", issue = "38356")]
Expand All @@ -799,43 +786,40 @@ impl Op {
/// Prints the punctuation character as a string that should be losslessly convertible
/// back into the same character.
#[unstable(feature = "proc_macro", issue = "38356")]
impl fmt::Display for Op {
impl fmt::Display for Punct {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
TokenStream::from(TokenTree::from(self.clone())).fmt(f)
}
}

/// An identifier (`ident`) or lifetime identifier (`'ident`).
///
/// REVIEW We should guarantee that `Term` contains a valid identifier permitted by
/// REVIEW We should guarantee that `Ident` contains a valid identifier permitted by
/// REVIEW the language and not a random unicode string, at least for a start.
///
/// REVIEW Maybe let's name it `Ident` instead of inventing a new term, it's named "identifier"
/// REVIEW everywhere in the compiler, including `ident` in `macro`/`macro_rules!` DSL.
///
/// REVIEW We need to support raw identifiers here (`r#ident`) or at least be future compatible
/// REVIEW with them. Currently they are supported using "string typing" - if string "r#ident" is
/// REVIEW passed to `Term::new` it will be interpreted as a raw identifier later on, we should add
/// REVIEW a field `is_raw` and a separate constructor for it (`Term::new_raw` or something) and
/// REVIEW passed to `Ident::new` it will be interpreted as a raw identifier later on, we should add
/// REVIEW a field `is_raw` and a separate constructor for it (`Ident::new_raw` or something) and
/// REVIEW keep it unstable until raw identifiers are stabilized.
///
/// REVIEW ATTENTION: `Copy` impl on a struct with private fields.
/// REVIEW Do we want to guarantee `Term` to be `Copy`?
/// REVIEW Do we want to guarantee `Ident` to be `Copy`?
#[derive(Copy, Clone, Debug)]
#[unstable(feature = "proc_macro", issue = "38356")]
pub struct Term {
pub struct Ident {
// REVIEW(INTERNAL) Symbol + Span is actually `ast::Ident`! We can use it here.
sym: Symbol,
span: Span,
}

#[unstable(feature = "proc_macro", issue = "38356")]
impl !Send for Term {}
impl !Send for Ident {}
#[unstable(feature = "proc_macro", issue = "38356")]
impl !Sync for Term {}
impl !Sync for Ident {}

impl Term {
/// Creates a new `Term` with the given `string` as well as the specified
impl Ident {
/// Creates a new `Ident` with the given `string` as well as the specified
/// `span`.
///
/// Note that `span`, currently in rustc, configures the hygiene information
Expand All @@ -856,8 +840,8 @@ impl Term {
/// REVIEW TO_DO Do input validation, the argument should be a valid identifier or
/// REVIEW lifetime identifier.
#[unstable(feature = "proc_macro", issue = "38356")]
pub fn new(string: &str, span: Span) -> Term {
Term {
pub fn new(string: &str, span: Span) -> Ident {
Ident {
sym: Symbol::intern(string),
span,
}
Expand All @@ -870,14 +854,14 @@ impl Term {
unsafe { &*(&*self.sym.as_str() as *const str) }
}

/// Returns the span of this `Term`, encompassing the entire string returned
/// Returns the span of this `Ident`, encompassing the entire string returned
/// by `as_str`.
#[unstable(feature = "proc_macro", issue = "38356")]
pub fn span(&self) -> Span {
self.span
}

/// Configures the span of this `Term`, possibly changing its hygiene context.
/// Configures the span of this `Ident`, possibly changing its hygiene context.
#[unstable(feature = "proc_macro", issue = "38356")]
pub fn set_span(&mut self, span: Span) {
self.span = span;
Expand All @@ -887,7 +871,7 @@ impl Term {
/// Prints the identifier as a string that should be losslessly convertible
/// back into the same identifier.
#[unstable(feature = "proc_macro", issue = "38356")]
impl fmt::Display for Term {
impl fmt::Display for Ident {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
self.sym.as_str().fmt(f)
}
Expand All @@ -896,7 +880,7 @@ impl fmt::Display for Term {
/// A literal string (`"hello"`), byte string (`b"hello"`),
/// character (`'a'`), byte character (`b'a'`), an integer or floating point number
/// with or without a suffix (`1`, `1u8`, `2.3`, `2.3f32`).
/// Boolean literals like `true` and `false` do not belong here, they are `Term`s.
/// Boolean literals like `true` and `false` do not belong here, they are `Ident`s.
#[derive(Clone, Debug)]
#[unstable(feature = "proc_macro", issue = "38356")]
pub struct Literal {
Expand Down Expand Up @@ -1182,15 +1166,15 @@ impl TokenTree {
})
}
macro_rules! op {
($a:expr) => (tt!(Op::new($a, op_kind)));
($a:expr) => (tt!(Punct::new($a, op_kind)));
($a:expr, $b:expr) => ({
stack.push(tt!(Op::new($b, op_kind)));
tt!(Op::new($a, Spacing::Joint))
stack.push(tt!(Punct::new($b, op_kind)));
tt!(Punct::new($a, Spacing::Joint))
});
($a:expr, $b:expr, $c:expr) => ({
stack.push(tt!(Op::new($c, op_kind)));
stack.push(tt!(Op::new($b, Spacing::Joint)));
tt!(Op::new($a, Spacing::Joint))
stack.push(tt!(Punct::new($c, op_kind)));
stack.push(tt!(Punct::new($b, Spacing::Joint)));
tt!(Punct::new($a, Spacing::Joint))
})
}

Expand Down Expand Up @@ -1243,25 +1227,25 @@ impl TokenTree {
Question => op!('?'),

Ident(ident, false) | Lifetime(ident) => {
tt!(Term::new(&ident.name.as_str(), Span(span)))
tt!(self::Ident::new(&ident.name.as_str(), Span(span)))
}
Ident(ident, true) => {
tt!(Term::new(&format!("r#{}", ident), Span(span)))
tt!(self::Ident::new(&format!("r#{}", ident), Span(span)))
}
Literal(lit, suffix) => tt!(self::Literal { lit, suffix, span: Span(span) }),
DocComment(c) => {
let style = comments::doc_comment_style(&c.as_str());
let stripped = comments::strip_doc_comment_decoration(&c.as_str());
let stream = vec![
tt!(Term::new("doc", Span(span))),
tt!(Op::new('=', Spacing::Alone)),
tt!(self::Ident::new("doc", Span(span))),
tt!(Punct::new('=', Spacing::Alone)),
tt!(self::Literal::string(&stripped)),
].into_iter().collect();
stack.push(tt!(Group::new(Delimiter::Bracket, stream)));
if style == ast::AttrStyle::Inner {
stack.push(tt!(Op::new('!', Spacing::Alone)));
stack.push(tt!(Punct::new('!', Spacing::Alone)));
}
tt!(Op::new('#', Spacing::Alone))
tt!(Punct::new('#', Spacing::Alone))
}

Interpolated(_) => {
Expand All @@ -1281,15 +1265,15 @@ impl TokenTree {
use syntax::parse::token::*;
use syntax::tokenstream::{TokenTree, Delimited};

let (op, kind, span) = match self {
self::TokenTree::Op(tt) => (tt.op(), tt.spacing(), tt.span()),
let (ch, kind, span) = match self {
self::TokenTree::Punct(tt) => (tt.as_char(), tt.spacing(), tt.span()),
self::TokenTree::Group(tt) => {
return TokenTree::Delimited(tt.span.0, Delimited {
delim: tt.delimiter.to_internal(),
tts: tt.stream.0.into(),
}).into();
},
self::TokenTree::Term(tt) => {
self::TokenTree::Ident(tt) => {
let ident = ast::Ident::new(tt.sym, tt.span.0);
let sym_str = tt.sym.to_string();
let token = if sym_str.starts_with("'") {
Expand Down Expand Up @@ -1337,7 +1321,7 @@ impl TokenTree {
}
};

let token = match op {
let token = match ch {
'=' => Eq,
'<' => Lt,
'>' => Gt,
Expand All @@ -1359,7 +1343,7 @@ impl TokenTree {
'#' => Pound,
'$' => Dollar,
'?' => Question,
_ => panic!("unsupported character {}", op),
_ => panic!("unsupported character {}", ch),
};

let tree = TokenTree::Token(span.0, token);
Expand Down

0 comments on commit 47d4089

Please sign in to comment.