Permalink
Browse files

Auto merge of #39087 - nrc:qquote-empty-delim, r=jseyfried

proc macros/qquote: Handle empty delimited tokens

r? @jseyfried
  • Loading branch information...
2 parents 4ce7acc + 0541997 commit 32da85148adacf1eca8ae2c45474144964ca0841 @bors bors committed Jan 17, 2017
@@ -60,14 +60,14 @@ pub fn qquote<'cx>(cx: &'cx mut ExtCtxt, sp: Span, tts: &[TokenTree])
struct QDelimited {
delim: token::DelimToken,
open_span: Span,
- tts: Vec<QTT>,
+ tts: Vec<Qtt>,
close_span: Span,
}
#[derive(Debug)]
-enum QTT {
+enum Qtt {
TT(TokenTree),
- QDL(QDelimited),
+ Delimited(QDelimited),
QIdent(TokenTree),
}
@@ -103,10 +103,10 @@ fn qquoter<'cx>(cx: &'cx mut ExtCtxt, ts: TokenStream) -> TokenStream {
}
}
-fn qquote_iter<'cx>(cx: &'cx mut ExtCtxt, depth: i64, ts: TokenStream) -> (Bindings, Vec<QTT>) {
+fn qquote_iter<'cx>(cx: &'cx mut ExtCtxt, depth: i64, ts: TokenStream) -> (Bindings, Vec<Qtt>) {
let mut depth = depth;
let mut bindings: Bindings = Vec::new();
- let mut output: Vec<QTT> = Vec::new();
+ let mut output: Vec<Qtt> = Vec::new();
let mut iter = ts.iter();
@@ -133,32 +133,32 @@ fn qquote_iter<'cx>(cx: &'cx mut ExtCtxt, depth: i64, ts: TokenStream) -> (Bindi
for b in bindings.clone() {
debug!("{:?} = {}", b.0, pprust::tts_to_string(&b.1.to_tts()[..]));
}
- output.push(QTT::QIdent(as_tt(Token::Ident(new_id.clone()))));
+ output.push(Qtt::QIdent(as_tt(Token::Ident(new_id.clone()))));
} else {
depth = depth - 1;
- output.push(QTT::TT(next.clone()));
+ output.push(Qtt::TT(next.clone()));
}
}
TokenTree::Token(_, Token::Ident(id)) if is_qquote(id) => {
depth = depth + 1;
}
TokenTree::Delimited(_, ref dl) => {
let br = qquote_iter(cx, depth, TokenStream::from_tts(dl.tts.clone().to_owned()));
- let mut bind_ = br.0;
- let res_ = br.1;
- bindings.append(&mut bind_);
+ let mut nested_bindings = br.0;
+ let nested = br.1;
+ bindings.append(&mut nested_bindings);
let new_dl = QDelimited {
delim: dl.delim,
open_span: dl.open_span,
- tts: res_,
+ tts: nested,
close_span: dl.close_span,
};
- output.push(QTT::QDL(new_dl));
+ output.push(Qtt::Delimited(new_dl));
}
t => {
- output.push(QTT::TT(t));
+ output.push(Qtt::TT(t));
}
}
}
@@ -188,9 +188,9 @@ fn unravel_concats(tss: Vec<TokenStream>) -> TokenStream {
output
}
-/// This converts the vector of QTTs into a seet of Bindings for construction and the main
+/// This converts the vector of Qtts into a set of Bindings for construction and the main
/// body as a TokenStream.
-fn convert_complex_tts<'cx>(cx: &'cx mut ExtCtxt, tts: Vec<QTT>) -> (Bindings, TokenStream) {
+fn convert_complex_tts<'cx>(cx: &'cx mut ExtCtxt, tts: Vec<Qtt>) -> (Bindings, TokenStream) {
let mut pushes: Vec<TokenStream> = Vec::new();
let mut bindings: Bindings = Vec::new();
@@ -203,28 +203,37 @@ fn convert_complex_tts<'cx>(cx: &'cx mut ExtCtxt, tts: Vec<QTT>) -> (Bindings, T
}
let next = next.unwrap();
match next {
- QTT::TT(TokenTree::Token(_, t)) => {
+ Qtt::TT(TokenTree::Token(_, t)) => {
let token_out = emit_token(t);
pushes.push(token_out);
}
// FIXME handle sequence repetition tokens
- QTT::QDL(qdl) => {
- debug!(" QDL: {:?} ", qdl.tts);
- let new_id = Ident::with_empty_ctxt(Symbol::gensym("qdl_tmp"));
- let mut cct_rec = convert_complex_tts(cx, qdl.tts);
- bindings.append(&mut cct_rec.0);
- bindings.push((new_id, cct_rec.1));
-
- let sep = build_delim_tok(qdl.delim);
-
- pushes.push(build_mod_call(
- vec![Ident::from_str("proc_macro_tokens"),
- Ident::from_str("build"),
- Ident::from_str("build_delimited")],
- concat(from_tokens(vec![Token::Ident(new_id)]), concat(lex(","), sep)),
- ));
+ Qtt::Delimited(qdl) => {
+ debug!(" Delimited: {:?} ", qdl.tts);
+ let fresh_id = Ident::with_empty_ctxt(Symbol::gensym("qdl_tmp"));
+ let (mut nested_bindings, nested_toks) = convert_complex_tts(cx, qdl.tts);
+
+ let body = if nested_toks.is_empty() {
+ assert!(nested_bindings.is_empty());
+ build_mod_call(vec![Ident::from_str("TokenStream"),
+ Ident::from_str("mk_empty")],
+ TokenStream::mk_empty())
+ } else {
+ bindings.append(&mut nested_bindings);
+ bindings.push((fresh_id, nested_toks));
+ TokenStream::from_tokens(vec![Token::Ident(fresh_id)])
+ };
+
+ let delimitiers = build_delim_tok(qdl.delim);
+
+ pushes.push(build_mod_call(vec![Ident::from_str("proc_macro_tokens"),
+ Ident::from_str("build"),
+ Ident::from_str("build_delimited")],
+ flatten(vec![body,
+ lex(","),
+ delimitiers].into_iter())));
}
- QTT::QIdent(t) => {
+ Qtt::QIdent(t) => {
pushes.push(TokenStream::from_tts(vec![t]));
pushes.push(TokenStream::mk_empty());
}
@@ -240,14 +249,8 @@ fn convert_complex_tts<'cx>(cx: &'cx mut ExtCtxt, tts: Vec<QTT>) -> (Bindings, T
// Utilities
/// Unravels Bindings into a TokenStream of `let` declarations.
-fn unravel(binds: Bindings) -> TokenStream {
- let mut output = TokenStream::mk_empty();
-
- for b in binds {
- output = concat(output, build_let(b.0, b.1));
- }
-
- output
+fn unravel(bindings: Bindings) -> TokenStream {
+ flatten(bindings.into_iter().map(|(a, b)| build_let(a, b)))
}
/// Checks if the Ident is `unquote`.
@@ -24,6 +24,19 @@ pub fn concat(ts1: TokenStream, ts2: TokenStream) -> TokenStream {
TokenStream::concat(ts1, ts2)
}
+/// Flatten a sequence of TokenStreams into a single TokenStream.
+pub fn flatten<T: Iterator<Item=TokenStream>>(mut iter: T) -> TokenStream {
+ match iter.next() {
+ Some(mut ts) => {
+ for next in iter {
+ ts = TokenStream::concat(ts, next);
+ }
+ ts
+ }
+ None => TokenStream::mk_empty()
+ }
+}
+
/// Checks if two identifiers have the same name, disregarding context. This allows us to
/// fake 'reserved' keywords.
// FIXME We really want `free-identifier-=?` (a la Dybvig 1993). von Tander 2007 is
@@ -0,0 +1,35 @@
+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#![feature(plugin)]
+#![feature(plugin_registrar)]
+#![feature(rustc_private)]
+#![plugin(proc_macro_plugin)]
+
+extern crate rustc_plugin;
+extern crate proc_macro_tokens;
+extern crate syntax;
+
+use syntax::ext::proc_macro_shim::prelude::*;
+use proc_macro_tokens::prelude::*;
+
+use rustc_plugin::Registry;
+
+#[plugin_registrar]
+pub fn plugin_registrar(reg: &mut Registry) {
+ reg.register_macro("hello", hello);
+}
+
+// This macro is not very interesting, but it does contain delimited tokens with
+// no content - `()` and `{}` - which has caused problems in the past.
+fn hello<'cx>(cx: &'cx mut ExtCtxt, sp: Span, tts: &[TokenTree]) -> Box<MacResult + 'cx> {
+ let output = qquote!({ fn hello() {} hello(); });
+ build_block_emitter(cx, sp, output)
+}
@@ -0,0 +1,22 @@
+// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// Test that a macro can emit delimiters with nothing inside - `()`, `{}`
+
+// aux-build:hello_macro.rs
+// ignore-stage1
+
+#![feature(plugin)]
+#![feature(rustc_private)]
+#![plugin(hello_macro)]
+
+fn main() {
+ hello!();
+}

0 comments on commit 32da851

Please sign in to comment.