Auto merge of #37321 - nrc:lib-proc-macro, r=@alexcrichton

Split up libproc_macro_plugin

Separate the plugin code from non-plugin code to break a potential cycle in crates.

This will allow us to merge the new libproc_macro_tokens into libproc_macro.

r? @alexcrichton

mk/crates.mk

@@ -60,8 +60,8 @@ RUSTC_CRATES := rustc rustc_typeck rustc_mir rustc_borrowck rustc_resolve rustc_
                 rustc_data_structures rustc_platform_intrinsics rustc_errors \
                 rustc_plugin rustc_metadata rustc_passes rustc_save_analysis \
                 rustc_const_eval rustc_const_math rustc_incremental proc_macro
-HOST_CRATES := syntax syntax_ext proc_macro_plugin syntax_pos $(RUSTC_CRATES) rustdoc fmt_macros \
-		flate arena graphviz log serialize
+HOST_CRATES := syntax syntax_ext proc_macro_tokens proc_macro_plugin syntax_pos $(RUSTC_CRATES) \
+		rustdoc fmt_macros flate arena graphviz log serialize
 TOOLS := compiletest rustdoc rustc rustbook error_index_generator
 
 DEPS_core :=
@@ -102,8 +102,9 @@ DEPS_test := std getopts term native:rust_test_helpers
 
 DEPS_syntax := std term serialize log arena libc rustc_bitflags rustc_unicode rustc_errors syntax_pos
 DEPS_syntax_ext := syntax syntax_pos rustc_errors fmt_macros proc_macro
-DEPS_proc_macro_plugin := syntax syntax_pos rustc_plugin log
 DEPS_syntax_pos := serialize
+DEPS_proc_macro_tokens := syntax syntax_pos log
+DEPS_proc_macro_plugin := syntax syntax_pos rustc_plugin log proc_macro_tokens
 
 DEPS_rustc_const_math := std syntax log serialize
 DEPS_rustc_const_eval := rustc_const_math rustc syntax log serialize \

src/libproc_macro_plugin/Cargo.toml

@@ -12,3 +12,4 @@ log = { path = "../liblog" }
 rustc_plugin = { path = "../librustc_plugin" }
 syntax = { path = "../libsyntax" }
 syntax_pos = { path = "../libsyntax_pos" }
+proc_macro_tokens = { path = "../libproc_macro_tokens" }

src/libproc_macro_plugin/lib.rs

@@ -13,43 +13,14 @@
 //! A library for procedural macro writers.
 //!
 //! ## Usage
-//! This package provides the `qquote!` macro for syntax creation, and the prelude
-//! (at libproc_macro::prelude) provides a number of operations:
-//! - `concat`, for concatenating two TokenStreams.
-//! - `ident_eq`, for checking if two identifiers are equal regardless of syntax context.
-//! - `str_to_token_ident`, for converting an `&str` into a Token.
-//! - `keyword_to_token_delim`, for converting a `parse::token::keywords::Keyword` into a
-//!    Token.
-//! - `build_delimited`, for creating a new TokenStream from an existing one and a delimiter
-//!    by wrapping the TokenStream in the delimiter.
-//! - `build_bracket_delimited`, `build_brace_delimited`, and `build_paren_delimited`, for
-//!    easing the above.
-//! - `build_empty_args`, which returns a TokenStream containing `()`.
-//! - `lex`, which takes an `&str` and returns the TokenStream it represents.
-//!
-//! The `qquote!` macro also imports `syntax::ext::proc_macro_shim::prelude::*`, so you
+//! This crate provides the `qquote!` macro for syntax creation.
+//!
+//! The `qquote!` macro imports `syntax::ext::proc_macro_shim::prelude::*`, so you
 //! will need to `extern crate syntax` for usage. (This is a temporary solution until more
-//! of the external API in libproc_macro is stabilized to support the token construction
+//! of the external API in libproc_macro_tokens is stabilized to support the token construction
 //! operations that the qausiquoter relies on.) The shim file also provides additional
 //! operations, such as `build_block_emitter` (as used in the `cond` example below).
 //!
-//! ## TokenStreams
-//!
-//! TokenStreams serve as the basis of the macro system. They are, in essence, vectors of
-//! TokenTrees, where indexing treats delimited values as a single term. That is, the term
-//! `even(a+c) && even(b)` will be indexibly encoded as `even | (a+c) | even | (b)` where,
-//! in reality, `(a+c)` is actually a decorated pointer to `a | + | c`.
-//!
-//! If a user has a TokenStream that is a single, delimited value, they can use
-//! `maybe_delimited` to destruct it and receive the internal vector as a new TokenStream
-//! as:
-//! ```
-//! `(a+c)`.maybe_delimited() ~> Some(a | + | c)`
-//! ```
-//!
-//! Check the TokenStream documentation for more information; the structure also provides
-//! cheap concatenation and slicing.
-//!
 //! ## Quasiquotation
 //!
 //! The quasiquoter creates output that, when run, constructs the tokenstream specified as
@@ -118,12 +89,11 @@
 extern crate rustc_plugin;
 extern crate syntax;
 extern crate syntax_pos;
+extern crate proc_macro_tokens;
 #[macro_use] extern crate log;
 
 mod qquote;
-pub mod build;
-pub mod parse;
-pub mod prelude;
+
 use qquote::qquote;
 
 use rustc_plugin::Registry;

src/libproc_macro_plugin/qquote.rs

@@ -24,12 +24,9 @@
 //! TokenStream that resembles the output syntax.
 //!
 
-extern crate rustc_plugin;
-extern crate syntax;
-extern crate syntax_pos;
+use proc_macro_tokens::build::*;
+use proc_macro_tokens::parse::lex;
 
-use build::*;
-use parse::lex;
 use qquote::int_build::*;
 
 use syntax::ast::Ident;
@@ -51,7 +48,7 @@ pub fn qquote<'cx>(cx: &'cx mut ExtCtxt, sp: Span, tts: &[TokenTree])
     let output = qquoter(cx, TokenStream::from_tts(tts.clone().to_owned()));
     debug!("\nQQ out: {}\n", pprust::tts_to_string(&output.to_tts()[..]));
     let imports = concat(lex("use syntax::ext::proc_macro_shim::prelude::*;"),
-                         lex("use proc_macro_plugin::prelude::*;"));
+                         lex("use proc_macro_tokens::prelude::*;"));
     build_block_emitter(cx, sp, build_brace_delimited(concat(imports, output)))
 }
 
@@ -219,7 +216,7 @@ fn convert_complex_tts<'cx>(cx: &'cx mut ExtCtxt, tts: Vec<QTT>) -> (Bindings, T
 
                 let sep = build_delim_tok(qdl.delim);
 
-                pushes.push(build_mod_call(vec![str_to_ident("proc_macro_plugin"),
+                pushes.push(build_mod_call(vec![str_to_ident("proc_macro_tokens"),
                                                str_to_ident("build"),
                                                str_to_ident("build_delimited")],
                                           concat(from_tokens(vec![Token::Ident(new_id)]),
@@ -264,11 +261,8 @@ fn is_qquote(id: Ident) -> bool {
 }
 
 mod int_build {
-    extern crate syntax;
-    extern crate syntax_pos;
-
-    use parse::*;
-    use build::*;
+    use proc_macro_tokens::build::*;
+    use proc_macro_tokens::parse::*;
 
     use syntax::ast::{self, Ident};
     use syntax::codemap::{DUMMY_SP};

src/libproc_macro_tokens/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+authors = ["The Rust Project Developers"]
+name = "proc_macro_tokens"
+version = "0.0.0"
+
+[lib]
+path = "lib.rs"
+crate-type = ["dylib"]
+
+[dependencies]
+syntax = { path = "../libsyntax" }
+syntax_pos = { path = "../libsyntax_pos" }
+log = { path = "../liblog" }