Expand quote! to allow for using all tokens and literals

To celebrate this, we now have a `stringy_math!` macro which is used in
documentation of the upcoming macro feature.

book/src/SUMMARY.md

@@ -25,6 +25,7 @@
 - [Closures](./closures.md)
 - [Asynchronous programming](./async.md)
   - [Streams](./streams.md)
+- [Macros](./macros.md)
 - [Advanced](./advanced.md)
   - [Safety](./safety.md)
   - [The stack](./the_stack.md)

book/src/macros.md

@@ -0,0 +1,57 @@
+# Macros
+
+Rune has (experimental) support for macros. These are functions which expand
+into code, and can be used by library writers to "extend the compiler".
+
+For now, the following type of macros are support:
+* Function-like macros expanding to items (functions, type declarations, ..).
+* Function-like macros expanding to expression (statements, blocks, async blocks, ..).
+
+Macros can currently only be defined natively. This is to get around the rather
+tricky issue that the code of a macro has to be available and runnable during
+compilation. Native modules have a benefit here, because they have to be defined
+at a time when they are available to the compiler.
+
+It also means we can re-use all the compiler infrastructure for Rune as a
+library for macro authors. Which is really nice.
+
+## Writing a native macro
+
+The following is the definition of the `stringy_math!` macro. Which is a macro
+that can be invoked on expressions.
+
+```rust,noplaypen
+{{#include ../../crates/rune-macros/src/stringy_math_macro.rs}}
+```
+
+A macro is added to a [`Module`] using the [`Module::macro_`] function.
+
+```rust,noplaypen
+pub fn module() -> Result<runestick::Module, runestick::ContextError> {
+    let mut module = runestick::Module::new(&["std", "experiments"]);
+    module.macro_(&["stringy_math"], stringy_math_macro::stringy_math)?;
+    Ok(module)
+}
+```
+
+With this module installed, we can now take `stringy_math!` for a spin.
+
+```rune
+{{#include ../../scripts/book/macros/stringy_math.rn}}
+```
+
+```text
+$> cargo run -- scripts/book/macros/stringy_math.rn -O macros=true --experimental
+200
+== () (2.9737ms)
+```
+
+Until macros are considered somewhat stable, they will be hidden behind the `-O
+macros=true` compiler option. This can be set programmatically in
+[`Options::macros`]. `--experimental` is an option to Rune CLI which adds the
+`std::experimental` module, which contains weird and experimental things like
+`stringy_math!`.
+
+[`Module`]: https://docs.rs/runestick/0/runestick/module/struct.Module.html
+[`Module::macro_`]: https://docs.rs/runestick/0/runestick/module/struct.Module.html#method.macro_
+[`Options::macros`]: https://docs.rs/rune/0/rune/struct.Options.html#method.macros

crates/rune-cli/src/main.rs

@@ -62,6 +62,7 @@ async fn main() -> Result<()> {
     let mut dump_functions = false;
     let mut dump_types = false;
     let mut help = false;
+    let mut experimental = false;
 
     let mut options = rune::Options::default();
 
@@ -89,6 +90,9 @@ async fn main() -> Result<()> {
             "--dump-types" => {
                 dump_types = true;
             }
+            "--experimental" => {
+                experimental = true;
+            }
             "-O" => {
                 let opt = match args.next() {
                     Some(opt) => opt,
@@ -126,13 +130,15 @@ async fn main() -> Result<()> {
         println!("  --dump-functions  - Dump available functions.");
         println!("  --dump-types      - Dump available types.");
         println!("  --no-linking      - Disable link time checks.");
+        println!("  --experimental    - Enabled experimental features.");
         println!();
         println!("Compiler options:");
         println!("  -O <option>       - Update the given compiler option.");
         println!();
         println!("Available <option> arguments:");
         println!("  memoize-instance-fn[=<true/false>] - Inline the lookup of an instance function where appropriate.");
         println!("  link-checks[=<true/false>] - Perform linker checks which makes sure that called functions exist.");
+        println!("  macros[=<true/false>] - Enable macros (experimental).");
         return Ok(());
     }
 
@@ -144,7 +150,11 @@ async fn main() -> Result<()> {
     };
 
     let mut context = rune::default_context()?;
-    context.install(&rune_macros::module()?)?;
+
+    if experimental {
+        context.install(&rune_macros::module()?)?;
+    }
+
     let context = Arc::new(context);
 
     let mut warnings = rune::Warnings::new();

crates/rune-macros/src/lib.rs

@@ -30,38 +30,19 @@
 //!
 //! Native macros for Rune.
 
-use rune::{MacroContext, TokenStream};
+use rune::ast;
+use rune::{MacroContext, Parser, TokenStream};
+
+mod stringy_math_macro;
 
 /// Implementation for the `passthrough!` macro.
 fn passthrough_impl(_: &mut MacroContext, stream: &TokenStream) -> runestick::Result<TokenStream> {
     Ok(stream.clone())
 }
 
-/// Implementation for the `test_add!` macro.
-fn test_add(ctx: &mut MacroContext, stream: &TokenStream) -> runestick::Result<TokenStream> {
-    use rune::ast;
-    use rune::Resolve as _;
-
-    let mut parser = rune::Parser::from_token_stream(stream);
-
-    let ident = parser.parse::<ast::Ident>()?;
-    let var = parser.parse::<ast::Ident>()?;
-    parser.parse_eof()?;
-
-    let ident = ident.resolve(ctx.storage(), ctx.source())?;
-
-    if ident != "please" {
-        return Err(runestick::Error::msg("you didn't ask nicely..."));
-    }
-
-    Ok(rune::quote!(ctx => || { #var + #var }))
-}
-
 /// Implementation for the `make_function!` macro.
 fn make_function(ctx: &mut MacroContext, stream: &TokenStream) -> runestick::Result<TokenStream> {
-    use rune::ast;
-
-    let mut parser = rune::Parser::from_token_stream(stream);
+    let mut parser = Parser::from_token_stream(stream);
 
     let ident = parser.parse::<ast::Ident>()?;
     let _ = parser.parse::<ast::Rocket>()?;
@@ -71,11 +52,12 @@ fn make_function(ctx: &mut MacroContext, stream: &TokenStream) -> runestick::Res
     Ok(rune::quote!(ctx => fn #ident() { #output }))
 }
 
-/// Construct the `http` module.
+/// Construct the `std::experimental` module, which contains experimental
+/// macros.
 pub fn module() -> Result<runestick::Module, runestick::ContextError> {
     let mut module = runestick::Module::new(&["std", "experiments"]);
     module.macro_(&["passthrough"], passthrough_impl)?;
-    module.macro_(&["test_add"], test_add)?;
+    module.macro_(&["stringy_math"], stringy_math_macro::stringy_math)?;
     module.macro_(&["make_function"], make_function)?;
     Ok(module)
 }

crates/rune-macros/src/stringy_math_macro.rs

@@ -0,0 +1,45 @@
+use rune::ast;
+use rune::Resolve as _;
+use rune::{quote, MacroContext, Parser, TokenStream};
+
+/// Implementation for the `stringy_math!` macro.
+pub(crate) fn stringy_math(
+    ctx: &mut MacroContext,
+    stream: &TokenStream,
+) -> runestick::Result<TokenStream> {
+    let mut parser = Parser::from_token_stream(stream);
+
+    let mut output = quote!(ctx => 0);
+
+    while !parser.is_eof()? {
+        let op = parser.parse::<ast::Ident>()?.macro_resolve(ctx)?;
+
+        match op.as_ref() {
+            "add" => {
+                let op = parser.parse::<ast::Expr>()?;
+                output = quote!(ctx => (#output) + #op);
+            }
+            "sub" => {
+                let op = parser.parse::<ast::Expr>()?;
+                output = quote!(ctx => (#output) - #op);
+            }
+            "div" => {
+                let op = parser.parse::<ast::Expr>()?;
+                output = quote!(ctx => (#output) / #op);
+            }
+            "mul" => {
+                let op = parser.parse::<ast::Expr>()?;
+                output = quote!(ctx => (#output) * #op);
+            }
+            other => {
+                return Err(runestick::Error::msg(format!(
+                    "unsupported operation `{}`",
+                    other
+                )));
+            }
+        }
+    }
+
+    parser.parse_eof()?;
+    Ok(output)
+}

crates/rune/src/ast/expr.rs

@@ -348,7 +348,7 @@ impl Expr {
             ast::Kind::Self_ => Self::Self_(parser.parse()?),
             ast::Kind::Select => Self::ExprSelect(parser.parse()?),
             ast::Kind::PipePipe | ast::Kind::Pipe => Self::ExprClosure(parser.parse()?),
-            ast::Kind::Label => {
+            ast::Kind::Label(..) => {
                 let label = Some((parser.parse::<ast::Label>()?, parser.parse::<ast::Colon>()?));
                 let token = parser.token_peek_eof()?;
 
@@ -368,7 +368,7 @@ impl Expr {
                     }
                 });
             }
-            ast::Kind::Hash => Self::LitObject(parser.parse()?),
+            ast::Kind::Pound => Self::LitObject(parser.parse()?),
             ast::Kind::Bang | ast::Kind::Amp | ast::Kind::Star => Self::ExprUnary(parser.parse()?),
             ast::Kind::While => Self::ExprWhile(parser.parse()?),
             ast::Kind::Loop => Self::ExprLoop(parser.parse()?),
@@ -608,8 +608,8 @@ impl Peek for Expr {
             ast::Kind::Async => true,
             ast::Kind::Self_ => true,
             ast::Kind::Select => true,
-            ast::Kind::Label => matches!(t2.map(|t| t.kind), Some(ast::Kind::Colon)),
-            ast::Kind::Hash => true,
+            ast::Kind::Label(..) => matches!(t2.map(|t| t.kind), Some(ast::Kind::Colon)),
+            ast::Kind::Pound => true,
             ast::Kind::Bang | ast::Kind::Amp | ast::Kind::Star => true,
             ast::Kind::While => true,
             ast::Kind::Loop => true,

crates/rune/src/ast/expr_break.rs

@@ -1,5 +1,4 @@
 use crate::ast;
-use crate::ast::{Kind, Token};
 use crate::error::ParseError;
 use crate::parser::Parser;
 use crate::traits::{Parse, Peek};
@@ -62,16 +61,16 @@ impl Parse for ExprBreakValue {
         let token = parser.token_peek_eof()?;
 
         Ok(match token.kind {
-            ast::Kind::Label => Self::Label(parser.parse()?),
+            ast::Kind::Label(..) => Self::Label(parser.parse()?),
             _ => Self::Expr(Box::new(parser.parse()?)),
         })
     }
 }
 
 impl Peek for ExprBreakValue {
-    fn peek(t1: Option<Token>, t2: Option<Token>) -> bool {
+    fn peek(t1: Option<ast::Token>, t2: Option<ast::Token>) -> bool {
         match t1.map(|t| t.kind) {
-            Some(Kind::Label) => true,
+            Some(ast::Kind::Label(..)) => true,
             _ => ast::Expr::peek(t1, t2),
         }
     }

crates/rune/src/ast/ident.rs

@@ -9,7 +9,7 @@ pub struct Ident {
     /// The kind of the identifier.
     pub token: ast::Token,
     /// The kind of the identifier.
-    pub kind: ast::IdentKind,
+    pub kind: ast::StringSource,
 }
 
 impl Ident {
@@ -26,7 +26,7 @@ impl Parse for Ident {
         match token.kind {
             ast::Kind::Ident(kind) => Ok(Self { token, kind }),
             _ => Err(ParseError::TokenMismatch {
-                expected: ast::Kind::Ident(ast::IdentKind::Source),
+                expected: ast::Kind::Ident(ast::StringSource::Text),
                 actual: token.kind,
                 span: token.span,
             }),
@@ -50,19 +50,23 @@ impl<'a> Resolve<'a> for Ident {
         let span = self.token.span;
 
         match self.kind {
-            ast::IdentKind::Source => {
+            ast::StringSource::Text => {
                 let ident = source
                     .source(span)
                     .ok_or_else(|| ParseError::BadSlice { span })?;
 
                 Ok(Cow::Borrowed(ident))
             }
-            ast::IdentKind::Synthetic(id) => {
+            ast::StringSource::Synthetic(id) => {
                 let ident = storage
-                    .get_ident(id)
-                    .ok_or_else(|| ParseError::BadIdentId { id, span })?;
+                    .get_string(id)
+                    .ok_or_else(|| ParseError::BadSyntheticId {
+                        kind: "label",
+                        id,
+                        span,
+                    })?;
 
-                Ok(Cow::Owned(ident))
+                Ok(Cow::Owned(ident.to_owned()))
             }
         }
     }