Auto merge of #35538 - cgswords:libproc_macro, r=nrc

Kicking off libproc_macro

This PR introduces `libproc_macro`, which is currently quite bare-bones (just a few macro construction tools and an initial `quote!` macro).

This PR also introduces a few test cases for it, and an additional `shim` file (at `src/libsyntax/ext/proc_macro_shim.rs` to allow a facsimile usage of Macros 2.0 *today*!

mk/crates.mk

@@ -60,7 +60,7 @@ RUSTC_CRATES := rustc rustc_typeck rustc_mir rustc_borrowck rustc_resolve rustc_
                 rustc_data_structures rustc_platform_intrinsics rustc_errors \
                 rustc_plugin rustc_metadata rustc_passes rustc_save_analysis \
                 rustc_const_eval rustc_const_math rustc_incremental
-HOST_CRATES := syntax syntax_ext syntax_pos $(RUSTC_CRATES) rustdoc fmt_macros \
+HOST_CRATES := syntax syntax_ext proc_macro syntax_pos $(RUSTC_CRATES) rustdoc fmt_macros \
 		flate arena graphviz rbml log serialize
 TOOLS := compiletest rustdoc rustc rustbook error_index_generator
 
@@ -100,6 +100,7 @@ DEPS_test := std getopts term native:rust_test_helpers
 
 DEPS_syntax := std term serialize log arena libc rustc_bitflags rustc_unicode rustc_errors syntax_pos
 DEPS_syntax_ext := syntax syntax_pos rustc_errors fmt_macros
+DEPS_proc_macro := syntax syntax_pos rustc_plugin log
 DEPS_syntax_pos := serialize
 
 DEPS_rustc_const_math := std syntax log serialize
@@ -114,8 +115,9 @@ DEPS_rustc_borrowck := rustc log graphviz syntax syntax_pos rustc_errors rustc_m
 DEPS_rustc_data_structures := std log serialize
 DEPS_rustc_driver := arena flate getopts graphviz libc rustc rustc_back rustc_borrowck \
                      rustc_typeck rustc_mir rustc_resolve log syntax serialize rustc_llvm \
-	             rustc_trans rustc_privacy rustc_lint rustc_plugin \
-                     rustc_metadata syntax_ext rustc_passes rustc_save_analysis rustc_const_eval \
+                     rustc_trans rustc_privacy rustc_lint rustc_plugin \
+                     rustc_metadata syntax_ext proc_macro \
+                     rustc_passes rustc_save_analysis rustc_const_eval \
                      rustc_incremental syntax_pos rustc_errors
 DEPS_rustc_errors := log libc serialize syntax_pos
 DEPS_rustc_lint := rustc log syntax syntax_pos rustc_const_eval

src/libproc_macro/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+authors = ["The Rust Project Developers"]
+name = "proc_macro"
+version = "0.0.0"
+
+[lib]
+name = "proc_macro"
+path = "lib.rs"
+crate-type = ["dylib"]
+
+[dependencies]
+log = { path = "../liblog" }
+rustc_plugin = { path = "../librustc_plugin" }
+syntax = { path = "../libsyntax" }
+syntax_pos = { path = "../libsyntax_pos" }

src/libproc_macro/build.rs

@@ -0,0 +1,89 @@
+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+extern crate syntax;
+extern crate syntax_pos;
+
+use syntax::ast::Ident;
+use syntax::codemap::DUMMY_SP;
+use syntax::parse::token::{self, Token, keywords, str_to_ident};
+use syntax::tokenstream::{self, TokenTree, TokenStream};
+use std::rc::Rc;
+
+/// A wrapper around `TokenStream::concat` to avoid extra namespace specification and
+/// provide TokenStream concatenation as a generic operator.
+pub fn concat(ts1: TokenStream, ts2: TokenStream) -> TokenStream {
+    TokenStream::concat(ts1, ts2)
+}
+
+/// Checks if two identifiers have the same name, disregarding context. This allows us to
+/// fake 'reserved' keywords.
+// FIXME We really want `free-identifier-=?` (a la Dybvig 1993). von Tander 2007 is
+// probably the easiest way to do that.
+pub fn ident_eq(tident: &TokenTree, id: Ident) -> bool {
+    let tid = match *tident {
+        TokenTree::Token(_, Token::Ident(ref id)) => id,
+        _ => {
+            return false;
+        }
+    };
+
+    tid.name == id.name
+}
+
+// ____________________________________________________________________________________________
+// Conversion operators
+
+/// Convert a `&str` into a Token.
+pub fn str_to_token_ident(s: &str) -> Token {
+    Token::Ident(str_to_ident(s))
+}
+
+/// Converts a keyword (from `syntax::parse::token::keywords`) into a Token that
+/// corresponds to it.
+pub fn keyword_to_token_ident(kw: keywords::Keyword) -> Token {
+    Token::Ident(str_to_ident(&kw.name().as_str()[..]))
+}
+
+// ____________________________________________________________________________________________
+// Build Procedures
+
+/// Generically takes a `ts` and delimiter and returns `ts` delimited by the specified
+/// delimiter.
+pub fn build_delimited(ts: TokenStream, delim: token::DelimToken) -> TokenStream {
+    let tts = ts.to_tts();
+    TokenStream::from_tts(vec![TokenTree::Delimited(DUMMY_SP,
+                                                    Rc::new(tokenstream::Delimited {
+                                                        delim: delim,
+                                                        open_span: DUMMY_SP,
+                                                        tts: tts,
+                                                        close_span: DUMMY_SP,
+                                                    }))])
+}
+
+/// Takes `ts` and returns `[ts]`.
+pub fn build_bracket_delimited(ts: TokenStream) -> TokenStream {
+    build_delimited(ts, token::DelimToken::Bracket)
+}
+
+/// Takes `ts` and returns `{ts}`.
+pub fn build_brace_delimited(ts: TokenStream) -> TokenStream {
+    build_delimited(ts, token::DelimToken::Brace)
+}
+
+/// Takes `ts` and returns `(ts)`.
+pub fn build_paren_delimited(ts: TokenStream) -> TokenStream {
+    build_delimited(ts, token::DelimToken::Paren)
+}
+
+/// Constructs `()`.
+pub fn build_empty_args() -> TokenStream {
+    build_paren_delimited(TokenStream::mk_empty())
+}

src/libproc_macro/lib.rs

@@ -0,0 +1,137 @@
+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+//! # Proc_Macro
+//!
+//! A library for procedural macro writers.
+//!
+//! ## Usage
+//! This package provides the `qquote!` macro for syntax creation, and the prelude
+//! (at libproc_macro::prelude) provides a number of operations:
+//! - `concat`, for concatenating two TokenStreams.
+//! - `ident_eq`, for checking if two identifiers are equal regardless of syntax context.
+//! - `str_to_token_ident`, for converting an `&str` into a Token.
+//! - `keyword_to_token_delim`, for converting a `parse::token::keywords::Keyword` into a
+//!    Token.
+//! - `build_delimited`, for creating a new TokenStream from an existing one and a delimiter
+//!    by wrapping the TokenStream in the delimiter.
+//! - `build_bracket_delimited`, `build_brace_delimited`, and `build_paren_delimited`, for
+//!    easing the above.
+//! - `build_empty_args`, which returns a TokenStream containing `()`.
+//! - `lex`, which takes an `&str` and returns the TokenStream it represents.
+//!
+//! The `qquote!` macro also imports `syntax::ext::proc_macro_shim::prelude::*`, so you
+//! will need to `extern crate syntax` for usage. (This is a temporary solution until more
+//! of the external API in libproc_macro is stabilized to support the token construction
+//! operations that the qausiquoter relies on.) The shim file also provides additional
+//! operations, such as `build_block_emitter` (as used in the `cond` example below).
+//!
+//! ## TokenStreams
+//!
+//! TokenStreams serve as the basis of the macro system. They are, in essence, vectors of
+//! TokenTrees, where indexing treats delimited values as a single term. That is, the term
+//! `even(a+c) && even(b)` will be indexibly encoded as `even | (a+c) | even | (b)` where,
+//! in reality, `(a+c)` is actually a decorated pointer to `a | + | c`.
+//!
+//! If a user has a TokenStream that is a single, delimited value, they can use
+//! `maybe_delimited` to destruct it and receive the internal vector as a new TokenStream
+//! as:
+//! ```
+//! `(a+c)`.maybe_delimited() ~> Some(a | + | c)`
+//! ```
+//!
+//! Check the TokenStream documentation for more information; the structure also provides
+//! cheap concatenation and slicing.
+//!
+//! ## Quasiquotation
+//!
+//! The quasiquoter creates output that, when run, constructs the tokenstream specified as
+//! input. For example, `qquote!(5 + 5)` will produce a program, that, when run, will
+//! construct the TokenStream `5 | + | 5`.
+//!
+//! ### Unquoting
+//!
+//! Unquoting is currently done as `unquote`, and works by taking the single next
+//! TokenTree in the TokenStream as the unquoted term. Ergonomically, `unquote(foo)` works
+//! fine, but `unquote foo` is also supported.
+//!
+//! A simple example might be:
+//!
+//!```
+//!fn double(tmp: TokenStream) -> TokenStream {
+//!    qquote!(unquote(tmp) * 2)
+//!}
+//!```
+//!
+//! ### Large Example: Implementing Scheme's `cond`
+//!
+//! Below is the full implementation of Scheme's `cond` operator.
+//!
+//! ```
+//! fn cond_rec(input: TokenStream) -> TokenStream {
+//!   if input.is_empty() { return quote!(); }
+//!
+//!   let next = input.slice(0..1);
+//!   let rest = input.slice_from(1..);
+//!
+//!   let clause : TokenStream = match next.maybe_delimited() {
+//!     Some(ts) => ts,
+//!     _ => panic!("Invalid input"),
+//!   };
+//!
+//!   // clause is ([test]) [rhs]
+//!   if clause.len() < 2 { panic!("Invalid macro usage in cond: {:?}", clause) }
+//!
+//!   let test: TokenStream = clause.slice(0..1);
+//!   let rhs: TokenStream = clause.slice_from(1..);
+//!
+//!   if ident_eq(&test[0], str_to_ident("else")) || rest.is_empty() {
+//!     quote!({unquote(rhs)})
+//!   } else {
+//!     quote!({if unquote(test) { unquote(rhs) } else { cond!(unquote(rest)) } })
+//!   }
+//! }
+//! ```
+//!
+
+#![crate_name = "proc_macro"]
+#![unstable(feature = "rustc_private", issue = "27812")]
+#![feature(plugin_registrar)]
+#![crate_type = "dylib"]
+#![crate_type = "rlib"]
+#![doc(html_logo_url = "https://www.rust-lang.org/logos/rust-logo-128x128-blk-v2.png",
+       html_favicon_url = "https://doc.rust-lang.org/favicon.ico",
+       html_root_url = "https://doc.rust-lang.org/nightly/")]
+#![cfg_attr(not(stage0), deny(warnings))]
+
+#![feature(staged_api)]
+#![feature(rustc_diagnostic_macros)]
+#![feature(rustc_private)]
+
+extern crate rustc_plugin;
+extern crate syntax;
+extern crate syntax_pos;
+#[macro_use] extern crate log;
+
+mod qquote;
+pub mod build;
+pub mod parse;
+pub mod prelude;
+use qquote::qquote;
+
+use rustc_plugin::Registry;
+
+// ____________________________________________________________________________________________
+// Main macro definition
+
+#[plugin_registrar]
+pub fn plugin_registrar(reg: &mut Registry) {
+    reg.register_macro("qquote", qquote);
+}

src/libproc_macro/parse.rs

@@ -0,0 +1,26 @@
+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+//! Parsing utilities for writing procedural macros.
+
+extern crate syntax;
+
+use syntax::parse::{ParseSess, filemap_to_tts};
+use syntax::tokenstream::TokenStream;
+
+/// Map a string to tts, using a made-up filename. For example, `lex(15)` will return a
+/// TokenStream containing the literal 15.
+pub fn lex(source_str: &str) -> TokenStream {
+    let ps = ParseSess::new();
+    TokenStream::from_tts(filemap_to_tts(&ps,
+                                         ps.codemap().new_filemap("procmacro_lex".to_string(),
+                                                                  None,
+                                                                  source_str.to_owned())))
+}

src/libproc_macro/prelude.rs

@@ -0,0 +1,12 @@
+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+pub use build::*;
+pub use parse::*;