Add experimental macro subsystem

crates/rune-cli/src/main.rs

@@ -143,7 +143,9 @@ async fn main() -> Result<()> {
         }
     };
 
-    let context = Arc::new(rune::default_context()?);
+    let mut context = rune::default_context()?;
+    let context = Arc::new(context);
+
     let mut warnings = rune::Warnings::new();
     let mut sources = rune::Sources::new();
 

crates/rune/src/ast/expr.rs

@@ -55,6 +55,8 @@ pub enum Expr {
     ExprMatch(ast::ExprMatch),
     /// A function call,
     ExprCall(ast::ExprCall),
+    /// A macro call,
+    ExprCallMacro(ast::ExprCallMacro),
     /// A field access on an expression.
     ExprFieldAccess(ast::ExprFieldAccess),
     /// A grouped expression.
@@ -148,6 +150,7 @@ impl Expr {
             Self::ExprIf(expr) => expr.span(),
             Self::ExprMatch(expr) => expr.span(),
             Self::ExprCall(expr) => expr.span(),
+            Self::ExprCallMacro(expr) => expr.span(),
             Self::ExprFieldAccess(expr) => expr.span(),
             Self::ExprGroup(expr) => expr.span(),
             Self::ExprUnary(expr) => expr.span(),
@@ -228,6 +231,12 @@ impl Expr {
             )?));
         }
 
+        if parser.peek::<ast::Bang>()? {
+            return Ok(Self::ExprCallMacro(ast::ExprCallMacro::parse_with_path(
+                parser, path,
+            )?));
+        }
+
         Ok(Self::Path(path))
     }
 

crates/rune/src/ast/expr_call_macro.rs

@@ -0,0 +1,90 @@
+use crate::ast;
+use crate::error::ParseError;
+use crate::parser::Parser;
+use crate::token_stream::TokenStream;
+use runestick::Span;
+
+/// A function call `<expr>!(<args>)`.
+#[derive(Debug, Clone)]
+pub struct ExprCallMacro {
+    /// The expression being called over.
+    pub path: ast::Path,
+    /// Bang operator `!`.
+    pub bang: ast::Bang,
+    /// Opening paren.
+    pub open: ast::Token,
+    /// The tokens provided to the macro.
+    pub stream: TokenStream,
+    /// Closing paren.
+    pub close: ast::Token,
+}
+
+impl ExprCallMacro {
+    /// Parse with an expression.
+    pub fn parse_with_path(parser: &mut Parser, path: ast::Path) -> Result<Self, ParseError> {
+        let bang: ast::Bang = parser.parse()?;
+
+        let mut level = 1;
+        let open = parser.token_next()?;
+
+        let delim = match open.kind {
+            ast::Kind::Open(delim) => delim,
+            kind => {
+                return Err(ParseError::ExpectedMacroDelimiter {
+                    span: open.span,
+                    actual: kind,
+                })
+            }
+        };
+
+        let close;
+
+        let mut stream = Vec::new();
+        let end;
+
+        loop {
+            let token = parser.token_next()?;
+
+            match token.kind {
+                ast::Kind::Open(..) => level += 1,
+                ast::Kind::Close(actual) => {
+                    level -= 1;
+
+                    if level == 0 {
+                        if actual != delim {
+                            return Err(ParseError::ExpectedMacroCloseDelimiter {
+                                span: open.span,
+                                actual: token.kind,
+                                expected: ast::Kind::Close(delim),
+                            });
+                        }
+
+                        end = Span::point(token.span.start);
+                        close = token;
+                        break;
+                    }
+                }
+                _ => (),
+            }
+
+            stream.push(token);
+        }
+
+        let default_span = bang.span().join(close.span);
+
+        Ok(Self {
+            bang,
+            path,
+            open,
+            stream: TokenStream::new(stream, default_span, end),
+            close,
+        })
+    }
+}
+
+impl ExprCallMacro {
+    /// Access the span of expression.
+    pub fn span(&self) -> Span {
+        self.path.span().join(self.close.span)
+    }
+}

crates/rune/src/ast/mod.rs

@@ -20,6 +20,7 @@ mod expr_binary;
 mod expr_block;
 mod expr_break;
 mod expr_call;
+mod expr_call_macro;
 mod expr_closure;
 mod expr_else;
 mod expr_else_if;
@@ -77,6 +78,7 @@ pub use self::expr_binary::{BinOp, ExprBinary};
 pub use self::expr_block::ExprBlock;
 pub use self::expr_break::{ExprBreak, ExprBreakValue};
 pub use self::expr_call::ExprCall;
+pub use self::expr_call_macro::ExprCallMacro;
 pub use self::expr_closure::ExprClosure;
 pub use self::expr_else::ExprElse;
 pub use self::expr_else_if::ExprElseIf;
@@ -160,6 +162,12 @@ macro_rules! decl_tokens {
                     }
                 }
             }
+
+            impl crate::IntoTokens for $parser {
+                fn into_tokens(&self, stream: &mut crate::TokenStream) {
+                    stream.push(self.token);
+                }
+            }
         )*
     }
 }
@@ -212,6 +220,7 @@ decl_tokens! {
     (Impl, "The `impl` keyword", Kind::Impl),
     (Mul, "Multiply `*` operator.", Kind::Star),
     (Mod, "The `mod` keyword.", Kind::Mod),
+    (Bang, "The `!` operator.", Kind::Bang),
 }
 
 impl<'a> Resolve<'a> for Ident {

crates/rune/src/ast/token.rs

@@ -10,6 +10,12 @@ pub struct Token {
     pub kind: Kind,
 }
 
+impl crate::IntoTokens for Token {
+    fn into_tokens(&self, stream: &mut crate::TokenStream) {
+        stream.push(*self);
+    }
+}
+
 /// The kind of a number literal.
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub enum NumberKind {
@@ -70,6 +76,8 @@ impl Delimiter {
 pub enum Kind {
     /// A `self` token.
     Self_,
+    /// A `macro` token.
+    Macro,
     /// An `fn` token.
     Fn,
     /// An `enum` token.
@@ -242,6 +250,7 @@ impl fmt::Display for Kind {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match *self {
             Self::Self_ => write!(f, "self")?,
+            Self::Macro => write!(f, "macro")?,
             Self::Fn => write!(f, "fn")?,
             Self::Enum => write!(f, "enum")?,
             Self::Struct => write!(f, "struct")?,
@@ -322,3 +331,12 @@ impl fmt::Display for Kind {
         Ok(())
     }
 }
+
+impl crate::IntoTokens for Kind {
+    fn into_tokens(&self, stream: &mut crate::TokenStream) {
+        stream.push(Token {
+            kind: *self,
+            span: stream.default_span(),
+        });
+    }
+}

crates/rune/src/compile/expr.rs

@@ -113,6 +113,10 @@ impl Compile<(&ast::Expr, Needs)> for Compiler<'_> {
             ast::Expr::LitTemplate(lit_template) => {
                 self.compile((lit_template, needs))?;
             }
+            ast::Expr::ExprCallMacro(expr_call_macro) => {
+                let expr: ast::Expr = self.compile_macro(expr_call_macro)?;
+                self.compile((&expr, needs))?;
+            }
             // NB: declarations are not used in this compilation stage.
             // They have been separately indexed and will be built when queried
             // for.

crates/rune/src/compiler.rs

@@ -2,14 +2,19 @@ use crate::assembly::Assembly;
 use crate::ast;
 use crate::collections::HashMap;
 use crate::error::CompileError;
-use crate::traits::{Compile as _, Resolve as _};
+use crate::parser::Parser;
+use crate::token_stream::TokenStream;
+use crate::traits::{Compile as _, Parse, Resolve as _};
 use crate::unit_builder::{ImportKey, UnitBuilder};
-use crate::SourceId;
-use runestick::{CompileMeta, Component, Context, Inst, Item, Label, Source, Span, TypeCheck};
+use crate::{MacroContext, SourceId};
+use runestick::{
+    CompileMeta, Component, Context, Hash, Inst, Item, Label, Source, Span, TypeCheck,
+};
 use std::cell::RefCell;
 use std::rc::Rc;
 
 use crate::error::CompileResult;
+use crate::error::ParseError;
 use crate::index::{Import, Index, Indexer};
 use crate::items::Items;
 use crate::load_error::{LoadError, LoadErrorKind};
@@ -99,9 +104,18 @@ pub fn compile_with_options(
     process_imports(imports, context, &mut *unit.borrow_mut())?;
 
     while let Some(entry) = query.queue.pop_front() {
+        let mut macro_context = MacroContext::new(entry.source.clone());
         let source_id = entry.source_id;
 
-        if let Err(error) = compile_entry(context, options, unit, warnings, &mut query, entry) {
+        if let Err(error) = compile_entry(
+            context,
+            &mut macro_context,
+            options,
+            unit,
+            warnings,
+            &mut query,
+            entry,
+        ) {
             return Err(LoadError::from(LoadErrorKind::CompileError {
                 source_id,
                 error,
@@ -114,6 +128,7 @@ pub fn compile_with_options(
 
 fn compile_entry(
     context: &Context,
+    macro_context: &mut MacroContext,
     options: &Options,
     unit: &Rc<RefCell<UnitBuilder>>,
     warnings: &mut Warnings,
@@ -133,6 +148,7 @@ fn compile_entry(
         source_id,
         source: source.clone(),
         context,
+        macro_context,
         query,
         asm: &mut asm,
         items: Items::new(item.as_vec()),
@@ -354,6 +370,8 @@ pub(crate) struct Compiler<'a> {
     pub(crate) source: Arc<Source>,
     /// The context we are compiling for.
     context: &'a Context,
+    /// Macro context, used for allocating space for identifiers and such.
+    macro_context: &'a mut MacroContext,
     /// Query system to compile required items.
     pub(crate) query: &'a mut Query,
     /// The assembly we are generating.
@@ -403,6 +421,64 @@ impl<'a> Compiler<'a> {
         Ok(None)
     }
 
+    /// Compile the given macro into the given output type.
+    pub(crate) fn compile_macro<T>(
+        &mut self,
+        expr_call_macro: &ast::ExprCallMacro,
+    ) -> CompileResult<T>
+    where
+        T: Parse,
+    {
+        let span = expr_call_macro.span();
+
+        if !self.options.macros {
+            return Err(CompileError::experimental(
+                "macros must be enabled with `-O macros=true`",
+                span,
+            ));
+        }
+
+        let item = self.convert_path_to_item(&expr_call_macro.path)?;
+        let hash = Hash::type_hash(&item);
+
+        let handler = match self.context.lookup_macro(hash) {
+            Some(handler) => handler,
+            None => {
+                return Err(CompileError::MissingMacro { span, item });
+            }
+        };
+
+        let input_stream = &expr_call_macro.stream;
+
+        let output = match handler(self.macro_context, input_stream) {
+            Ok(output) => output,
+            Err(error) => {
+                return match error.downcast::<ParseError>() {
+                    Ok(error) => Err(CompileError::ParseError { error }),
+                    Err(error) => Err(CompileError::CallMacroError { span, error }),
+                };
+            }
+        };
+
+        let token_stream = match output.downcast::<TokenStream>() {
+            Ok(token_stream) => *token_stream,
+            Err(..) => {
+                return Err(CompileError::CallMacroError {
+                    span,
+                    error: runestick::Error::msg(format!(
+                        "failed to downcast macro result, expected `{}`",
+                        std::any::type_name::<TokenStream>()
+                    )),
+                });
+            }
+        };
+
+        let mut parser = Parser::from_token_stream(&token_stream);
+        let output = parser.parse::<T>()?;
+        parser.parse_eof()?;
+        Ok(output)
+    }
+
     /// Pop locals by simply popping them.
     pub(crate) fn locals_pop(&mut self, total_var_count: usize, span: Span) {
         match total_var_count {