diff --git a/crates/mq-lang/src/ast/node.rs b/crates/mq-lang/src/ast/node.rs index 92fd3987..30e3ea66 100644 --- a/crates/mq-lang/src/ast/node.rs +++ b/crates/mq-lang/src/ast/node.rs @@ -106,7 +106,7 @@ impl Node { .unwrap_or_else(|| arena[self.token_id].range.end.clone()); Range { start, end } } - Expr::Paren(node) => node.range(Shared::clone(&arena)), + Expr::Paren(node) | Expr::Module(node) => node.range(Shared::clone(&arena)), Expr::Try(try_expr, catch_expr) => { let start = try_expr.range(Shared::clone(&arena)).start; let end = catch_expr.range(Shared::clone(&arena)).end; @@ -116,6 +116,7 @@ impl Node { | Expr::Ident(_) | Expr::Selector(_) | Expr::Include(_) + | Expr::Import(_, _) | Expr::InterpolatedString(_) | Expr::Nodes | Expr::Self_ @@ -294,6 +295,8 @@ pub enum Expr { If(Branches), Match(Shared, MatchArms), Include(Literal), + Import(Literal, Option), + Module(Shared), Self_, Nodes, Paren(Shared), diff --git a/crates/mq-lang/src/ast/parser.rs b/crates/mq-lang/src/ast/parser.rs index 49b599bf..e80541b7 100644 --- a/crates/mq-lang/src/ast/parser.rs +++ b/crates/mq-lang/src/ast/parser.rs @@ -79,6 +79,13 @@ impl<'a, 'alloc> Parser<'a, 'alloc> { TokenKind::Nodes => { return Err(ParseError::UnexpectedToken((**token).clone())); } + TokenKind::Module if root => { + let ast = self.parse_module(Shared::clone(token))?; + asts.push(ast); + } + TokenKind::Module => { + return Err(ParseError::UnexpectedToken((**token).clone())); + } TokenKind::NewLine | TokenKind::Tab(_) | TokenKind::Whitespace(_) => unreachable!(), _ => { let ast = self.parse_expr(Shared::clone(token))?; @@ -219,6 +226,7 @@ impl<'a, 'alloc> Parser<'a, 'alloc> { TokenKind::Match => self.parse_expr_match(Shared::clone(&token)), TokenKind::InterpolatedString(_) => self.parse_interpolated_string(token), TokenKind::Include => self.parse_include(token), + TokenKind::Import => self.parse_import(token), TokenKind::Self_ => self.parse_self(token), TokenKind::Break => self.parse_break(token), TokenKind::Continue => self.parse_continue(token), @@ -239,6 +247,24 @@ impl<'a, 'alloc> Parser<'a, 'alloc> { } } + fn parse_module(&mut self, token: Shared) -> Result, ParseError> { + match &token.kind { + TokenKind::Module => { + let next_token = match self.tokens.next() { + Some(t) => t, + None => return Err(ParseError::UnexpectedEOFDetected(self.module_id)), + }; + let metadata = self.parse_dict(Shared::clone(next_token))?; + + Ok(Shared::new(Node { + token_id: self.token_arena.alloc(token), + expr: Shared::new(Expr::Module(metadata)), + })) + } + _ => Err(ParseError::UnexpectedToken((*token).clone())), + } + } + fn parse_symbol(&mut self, token: Shared) -> Result, ParseError> { match &token.kind { TokenKind::Colon => { @@ -1359,6 +1385,39 @@ impl<'a, 'alloc> Parser<'a, 'alloc> { } } + #[inline(always)] + fn parse_import(&mut self, import_token: Shared) -> Result, ParseError> { + let token_id = self.token_arena.alloc(import_token); + let token = match self.tokens.peek() { + Some(token) => Ok(Shared::clone(token)), + None => Err(ParseError::UnexpectedEOFDetected(self.module_id)), + }?; + + match &token.kind { + TokenKind::StringLiteral(module) => { + self.tokens.next(); + + match self.tokens.peek() { + Some(token) => match &token.kind { + TokenKind::StringLiteral(module) => Ok(Shared::new(Node { + token_id, + expr: Shared::new(Expr::Import( + Literal::String(module.to_owned()), + None, + )), + })), + _ => Err(ParseError::InsufficientTokens((***token).clone())), + }, + None => Ok(Shared::new(Node { + token_id, + expr: Shared::new(Expr::Import(Literal::String(module.to_owned()), None)), + })), + } + } + _ => Err(ParseError::InsufficientTokens((*token).clone())), + } + } + fn parse_interpolated_string( &mut self, token: Shared, @@ -5490,6 +5549,47 @@ mod tests { token(TokenKind::Eof) ], Err(ParseError::UnexpectedEOFDetected(Module::TOP_LEVEL_MODULE_ID)))] + #[case::module_simple( + vec![ + token(TokenKind::Module), + token(TokenKind::LBrace), + token(TokenKind::StringLiteral("key".to_owned())), + token(TokenKind::Colon), + token(TokenKind::StringLiteral("value".to_owned())), + token(TokenKind::RBrace), + token(TokenKind::Eof) + ], + Ok(vec![ + Shared::new(Node { + token_id: 0.into(), + expr: Shared::new(Expr::Module( + Shared::new(Node { + token_id: 0.into(), + expr: Shared::new(Expr::Call( + IdentWithToken::new_with_token(constants::DICT, Some(Shared::new(token(TokenKind::LBrace)))), + smallvec![ + Shared::new(Node { + token_id: 0.into(), + expr: Shared::new(Expr::Call( + IdentWithToken::new_with_token(constants::ARRAY, Some(Shared::new(token(TokenKind::StringLiteral("key".to_owned()))))), + smallvec![ + Shared::new(Node { + token_id: 1.into(), + expr: Shared::new(Expr::Literal(Literal::String("key".to_owned()))), + }), + Shared::new(Node { + token_id: 2.into(), + expr: Shared::new(Expr::Literal(Literal::String("value".to_owned()))), + }), + ], + )), + }), + ], + )), + }), + )), + }) + ]))] fn test_parse(#[case] input: Vec, #[case] expected: Result) { let mut arena = Arena::new(10); let tokens: Vec> = input.into_iter().map(Shared::new).collect(); diff --git a/crates/mq-lang/src/eval.rs b/crates/mq-lang/src/eval.rs index c76a8d7b..cebad722 100644 --- a/crates/mq-lang/src/eval.rs +++ b/crates/mq-lang/src/eval.rs @@ -156,6 +156,9 @@ impl Evaluator { ast::Expr::Include(module_id) => { self.eval_include(module_id.to_owned())?; } + ast::Expr::Import(module_path, alias) => { + self.eval_import(module_path.to_owned(), alias.to_owned())?; + } _ => nodes.push(Shared::clone(node)), }; @@ -219,9 +222,9 @@ impl Evaluator { Ok(match value { RuntimeValue::None => child_node.to_fragment(), - RuntimeValue::Function(_, _, _) | RuntimeValue::NativeFunction(_) => { - mq_markdown::Node::Empty - } + RuntimeValue::Function(_, _, _) + | RuntimeValue::NativeFunction(_) + | RuntimeValue::Module(_) => mq_markdown::Node::Empty, RuntimeValue::Array(_) | RuntimeValue::Dict(_) | RuntimeValue::Boolean(_) @@ -390,6 +393,97 @@ impl Evaluator { } } + fn eval_import( + &mut self, + module_path: ast::Literal, + alias: Option, + ) -> Result<(), EvalError> { + match module_path { + ast::Literal::String(module_name) => { + let module = self + .module_loader + .load_from_file(&module_name, Shared::clone(&self.token_arena))?; + + if let Some(module) = module { + // Create a new environment for the module exports + let module_env = Shared::new(SharedCell::new(Env::with_parent( + Shared::downgrade(&self.env), + ))); + + // Load module contents into the module environment + for node in &module.modules { + if let ast::Expr::Include(_) = &*node.expr { + self.eval_expr(&RuntimeValue::NONE, node, &Shared::clone(&module_env))?; + } + } + + for node in &module.functions { + if let ast::Expr::Def(ident, params, program) = &*node.expr { + define( + &module_env, + ident.name, + RuntimeValue::Function( + params.clone(), + program.clone(), + Shared::clone(&module_env), + ), + ); + } + } + + for node in &module.vars { + if let ast::Expr::Let(ident, node) = &*node.expr { + let val = self.eval_expr( + &RuntimeValue::NONE, + node, + &Shared::clone(&module_env), + )?; + define(&module_env, ident.name, val); + } + } + + let module_name_to_use = match alias { + Some(ast::Literal::String(alias_name)) => alias_name, + _ => { + if let Some(metadata) = module.metadata { + match self.eval_expr( + &RuntimeValue::NONE, + &metadata, + &Shared::clone(&module_env), + )? { + RuntimeValue::Dict(dict) => dict + .get(&Ident::new("name")) + .map(|name| name.to_string()) + .unwrap_or(module.name), + _ => module.name, + } + } else { + module.name + } + } + }; + + // Register the module in the environment + let module_runtime_value = RuntimeValue::Module(runtime_value::ModuleEnv::new( + &module_name_to_use, + module_env, + )); + + define( + &self.env, + Ident::new(&module_name_to_use), + module_runtime_value, + ); + } + + Ok(()) + } + _ => Err(EvalError::ModuleLoadError( + module::ModuleError::InvalidModule, + )), + } + } + #[inline(always)] fn eval_selector_expr(runtime_value: &RuntimeValue, ident: &ast::Selector) -> RuntimeValue { match runtime_value { @@ -577,6 +671,13 @@ impl Evaluator { self.eval_include(module_id.to_owned())?; Ok(runtime_value.clone()) } + ast::Expr::Import(module_path, alias) => { + self.eval_import(module_path.to_owned(), alias.to_owned())?; + Ok(runtime_value.clone()) + } + ast::Expr::Module(_module) => { + todo!() + } ast::Expr::Match(value_node, arms) => { self.eval_match(runtime_value, value_node, arms, env) } diff --git a/crates/mq-lang/src/eval/env.rs b/crates/mq-lang/src/eval/env.rs index 12e59aaf..340e8563 100644 --- a/crates/mq-lang/src/eval/env.rs +++ b/crates/mq-lang/src/eval/env.rs @@ -101,6 +101,11 @@ impl Variable { value: "native function".to_string(), type_field: "native_function".to_string(), }, + RuntimeValue::Module(m) => Variable { + name: m.name().to_string(), + value: format!("module/{}", m.len()), + type_field: "module".to_string(), + }, RuntimeValue::None => Variable { name: ident.to_string(), value: "None".to_string(), @@ -129,6 +134,10 @@ impl Env { } } + pub fn len(&self) -> usize { + self.context.len() + } + #[inline(always)] pub fn define(&mut self, ident: Ident, runtime_value: RuntimeValue) { self.context.insert(ident, runtime_value); diff --git a/crates/mq-lang/src/eval/module.rs b/crates/mq-lang/src/eval/module.rs index cd9dacbe..b141c2d2 100644 --- a/crates/mq-lang/src/eval/module.rs +++ b/crates/mq-lang/src/eval/module.rs @@ -50,6 +50,7 @@ pub struct Module { pub functions: Program, pub modules: Program, pub vars: Program, + pub metadata: Option>, } impl Module { @@ -156,6 +157,11 @@ impl ModuleLoader { .cloned() .collect::>(); + let metadata = program + .iter() + .find(|node| matches!(*node.expr, ast::Expr::Module(_))) + .cloned(); + if program.len() != functions.len() + modules.len() + vars.len() { return Err(ModuleError::InvalidModule); } @@ -165,6 +171,7 @@ impl ModuleLoader { functions, modules, vars, + metadata, })) } @@ -338,7 +345,8 @@ mod tests { module_id: 1.into() }))), Shared::new(ast::Node{token_id: 2.into(), expr: Shared::new(ast::Expr::Literal(ast::Literal::String("value".to_string())))}) - ))})] + ))})], + metadata: None })))] #[case::load3("def test(): 1;".to_string(), Ok(Some(Module{ name: "test".to_string(), @@ -355,7 +363,8 @@ mod tests { Shared::new(ast::Node{token_id: 2.into(), expr: Shared::new(ast::Expr::Literal(ast::Literal::Number(1.into())))}) ] ))})], - vars: Vec::new() + vars: Vec::new(), + metadata: None })))] #[case::load4("def test(a, b): add(a, b);".to_string(), Ok(Some(Module{ name: "test".to_string(), @@ -388,7 +397,8 @@ mod tests { ], ))})] ))})], - vars: Vec::new() + vars: Vec::new(), + metadata: None })))] fn test_load( token_arena: Shared>>>, @@ -407,6 +417,7 @@ mod tests { functions: Vec::new(), modules: Vec::new(), // Assuming the csv.mq only contains definitions or is empty for this test vars: Vec::new(), + metadata: None })))] fn test_load_standard_module( token_arena: Shared>>>, diff --git a/crates/mq-lang/src/eval/runtime_value.rs b/crates/mq-lang/src/eval/runtime_value.rs index e5c6e118..fa4f908c 100644 --- a/crates/mq-lang/src/eval/runtime_value.rs +++ b/crates/mq-lang/src/eval/runtime_value.rs @@ -1,6 +1,7 @@ use super::env::Env; use crate::{AstParams, Ident, Program, Shared, SharedCell, number::Number}; use mq_markdown::Node; +use smol_str::SmolStr; use std::{ borrow::Cow, cmp::Ordering, @@ -13,6 +14,37 @@ pub enum Selector { Index(usize), } +#[derive(Clone, Debug)] +pub struct ModuleEnv { + name: SmolStr, + exports: Shared>, +} + +impl ModuleEnv { + pub fn new(name: &str, exports: Shared>) -> Self { + Self { + name: SmolStr::new(name), + exports, + } + } + + pub fn name(&self) -> &str { + &self.name + } + + pub fn len(&self) -> usize { + #[cfg(not(feature = "sync"))] + { + self.exports.borrow().len() + } + + #[cfg(feature = "sync")] + { + self.exports.read().unwrap().len() + } + } +} + #[derive(Clone, Default)] pub enum RuntimeValue { Number(Number), @@ -24,6 +56,7 @@ pub enum RuntimeValue { Function(AstParams, Program, Shared>), NativeFunction(Ident), Dict(BTreeMap), + Module(ModuleEnv), #[default] None, } @@ -43,6 +76,7 @@ impl PartialEq for RuntimeValue { } (RuntimeValue::NativeFunction(a), RuntimeValue::NativeFunction(b)) => a == b, (RuntimeValue::Dict(a), RuntimeValue::Dict(b)) => a == b, + (RuntimeValue::Module(a), RuntimeValue::Module(b)) => std::ptr::eq(a, b), (RuntimeValue::None, RuntimeValue::None) => true, _ => false, } @@ -154,6 +188,7 @@ impl PartialOrd for RuntimeValue { } (RuntimeValue::Dict(_), _) => None, (_, RuntimeValue::Dict(_)) => None, + (RuntimeValue::Module(a), RuntimeValue::Module(b)) => a.name.partial_cmp(&b.name), _ => None, } } @@ -172,6 +207,7 @@ impl std::fmt::Display for RuntimeValue { Self::Function(params, ..) => Cow::Owned(format!("function/{}", params.len())), Self::NativeFunction(_) => Cow::Borrowed("native_function"), Self::Dict(_) => self.string(), + Self::Module(module_name) => Cow::Owned(format!(r#"module "{}""#, module_name.name)), }; write!(f, "{}", value) } @@ -211,6 +247,7 @@ impl RuntimeValue { RuntimeValue::Function(_, _, _) => "function", RuntimeValue::NativeFunction(_) => "native_function", RuntimeValue::Dict(_) => "dict", + RuntimeValue::Module(_) => "module", } } @@ -261,6 +298,7 @@ impl RuntimeValue { | RuntimeValue::Function(_, _, _) | RuntimeValue::NativeFunction(_) | RuntimeValue::Dict(_) => true, + RuntimeValue::Module(_) => true, RuntimeValue::None => false, } } @@ -277,6 +315,7 @@ impl RuntimeValue { RuntimeValue::Dict(m) => m.len(), RuntimeValue::None => 0, RuntimeValue::Function(..) => 0, + RuntimeValue::Module(m) => m.len(), RuntimeValue::NativeFunction(..) => 0, } } @@ -334,8 +373,9 @@ impl RuntimeValue { )), Self::Markdown(m, ..) => Cow::Owned(m.to_string()), Self::None => Cow::Borrowed(""), - Self::Function(..) => Cow::Borrowed("function"), + Self::Function(f, _, _) => Cow::Owned(format!("function/{}", f.len())), Self::NativeFunction(_) => Cow::Borrowed("native_function"), + Self::Module(m) => Cow::Owned(format!("module/{}", m.name())), Self::Dict(map) => { let items = map .iter() @@ -413,6 +453,7 @@ impl RuntimeValues { match &updated_value { RuntimeValue::None | RuntimeValue::Function(_, _, _) + | RuntimeValue::Module(_) | RuntimeValue::NativeFunction(_) => current_value.clone(), RuntimeValue::Markdown(node, _) if node.is_empty() => current_value.clone(), RuntimeValue::Markdown(node, _) => { @@ -727,7 +768,7 @@ mod tests { Vec::new(), Shared::new(SharedCell::new(Env::default())), ); - assert_eq!(format!("{:?}", function), "function"); + assert_eq!(format!("{:?}", function), "function/0"); let native_fn = RuntimeValue::NativeFunction(Ident::new("debug")); assert_eq!(format!("{:?}", native_fn), "native_function"); diff --git a/crates/mq-lang/src/lexer.rs b/crates/mq-lang/src/lexer.rs index d0582181..a5d0fb27 100644 --- a/crates/mq-lang/src/lexer.rs +++ b/crates/mq-lang/src/lexer.rs @@ -209,10 +209,12 @@ fn spaces(input: Span) -> IResult { .parse(input) } +define_keyword_parser!(as_, "as", TokenKind::As); define_token_parser!(colon, ":", TokenKind::Colon); define_token_parser!(comma, ",", TokenKind::Comma); define_keyword_parser!(def, "def", TokenKind::Def); define_keyword_parser!(do_, "do", TokenKind::Do); +define_token_parser!(double_colon, "::", TokenKind::DoubleColon); define_keyword_parser!(elif, "elif", TokenKind::Elif); define_keyword_parser!(else_, "else", TokenKind::Else); define_keyword_parser!(end, "end", TokenKind::End); @@ -229,6 +231,7 @@ define_keyword_parser!(fn_, "fn", TokenKind::Fn); define_keyword_parser!(foreach, "foreach", TokenKind::Foreach); define_keyword_parser!(if_, "if", TokenKind::If); define_keyword_parser!(include, "include", TokenKind::Include); +define_keyword_parser!(import, "import", TokenKind::Import); define_token_parser!(l_bracket, "[", TokenKind::LBracket); define_token_parser!(l_paren, "(", TokenKind::LParen); define_token_parser!(l_brace, "{", TokenKind::LBrace); @@ -265,8 +268,21 @@ define_token_parser!(coalesce, "??", TokenKind::Coalesce); fn punctuations(input: Span) -> IResult { alt(( - and, or, l_paren, r_paren, l_brace, r_brace, comma, colon, semi_colon, l_bracket, - r_bracket, coalesce, question, pipe, + and, + or, + l_paren, + r_paren, + l_brace, + r_brace, + comma, + colon, + double_colon, + semi_colon, + l_bracket, + r_bracket, + coalesce, + question, + pipe, )) .parse(input) } @@ -282,14 +298,22 @@ fn unary_op(input: Span) -> IResult { alt((not,)).parse(input) } -fn keywords(input: Span) -> IResult { +fn control_keywords(input: Span) -> IResult { alt(( - nodes, def, do_, let_, match_, self_, while_, until, if_, elif, else_, end, none, include, - foreach, fn_, break_, continue_, try_, catch_, + def, do_, let_, match_, while_, until, if_, elif, else_, end, foreach, fn_, break_, + continue_, try_, catch_, )) .parse(input) } +fn builtin_keywords(input: Span) -> IResult { + alt((nodes, self_, none, include, import, as_)).parse(input) +} + +fn keywords(input: Span) -> IResult { + alt((control_keywords, builtin_keywords)).parse(input) +} + fn number_literal(input: Span) -> IResult { map_res( recognize(pair( diff --git a/crates/mq-lang/src/lexer/token.rs b/crates/mq-lang/src/lexer/token.rs index e157b1a5..689dc709 100644 --- a/crates/mq-lang/src/lexer/token.rs +++ b/crates/mq-lang/src/lexer/token.rs @@ -49,12 +49,14 @@ pub struct Token { /// TokenKind variants are sorted alphabetically for maintainability. pub enum TokenKind { And, + As, Asterisk, BoolLiteral(bool), Break, Catch, Coalesce, Colon, + DoubleColon, Comma, Comment(String), Continue, @@ -75,12 +77,14 @@ pub enum TokenKind { If, Include, InterpolatedString(Vec), + Import, LBrace, LBracket, Let, Lt, Lte, Match, + Module, Minus, NeEq, NewLine, @@ -126,6 +130,7 @@ impl Display for TokenKind { fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), fmt::Error> { match &self { TokenKind::And => write!(f, "&&"), + TokenKind::As => write!(f, "as"), TokenKind::Or => write!(f, "||"), TokenKind::Not => write!(f, "!"), TokenKind::Asterisk => write!(f, "*"), @@ -138,6 +143,7 @@ impl Display for TokenKind { TokenKind::Comment(comment) => write!(f, "# {}", comment.trim()), TokenKind::Def => write!(f, "def"), TokenKind::Do => write!(f, "do"), + TokenKind::DoubleColon => write!(f, "::"), TokenKind::Elif => write!(f, "elif"), TokenKind::Else => write!(f, "else"), TokenKind::End => write!(f, "end"), @@ -150,6 +156,7 @@ impl Display for TokenKind { TokenKind::Ident(ident) => write!(f, "{}", ident), TokenKind::If => write!(f, "if"), TokenKind::Include => write!(f, "include"), + TokenKind::Import => write!(f, "import"), TokenKind::InterpolatedString(segments) => { write!(f, "{}", segments.iter().join("")) } @@ -161,6 +168,7 @@ impl Display for TokenKind { TokenKind::LParen => write!(f, "("), TokenKind::Let => write!(f, "let"), TokenKind::Match => write!(f, "match"), + TokenKind::Module => write!(f, "module"), TokenKind::Minus => write!(f, "-"), TokenKind::Slash => write!(f, "/"), TokenKind::Percent => write!(f, "%"), diff --git a/crates/mq-lang/src/optimizer.rs b/crates/mq-lang/src/optimizer.rs index c720afc4..f9d3aa03 100644 --- a/crates/mq-lang/src/optimizer.rs +++ b/crates/mq-lang/src/optimizer.rs @@ -179,6 +179,8 @@ impl Optimizer { | ast::Expr::Nodes | ast::Expr::Self_ | ast::Expr::Include(_) + | ast::Expr::Import(_, _) + | ast::Expr::Module(_) | ast::Expr::Break | ast::Expr::Continue => {} } diff --git a/crates/mq-python/src/value.rs b/crates/mq-python/src/value.rs index d2a938da..fb03e0b9 100644 --- a/crates/mq-python/src/value.rs +++ b/crates/mq-python/src/value.rs @@ -130,11 +130,9 @@ impl From for MQValue { value: b.to_string(), markdown_type: MarkdownType::Text, }, - mq_lang::RuntimeValue::Function(..) => MQValue::Markdown { - value: "".to_string(), - markdown_type: MarkdownType::Empty, - }, - mq_lang::RuntimeValue::NativeFunction(..) => MQValue::Markdown { + mq_lang::RuntimeValue::Function(..) + | mq_lang::RuntimeValue::NativeFunction(..) + | mq_lang::RuntimeValue::Module(..) => MQValue::Markdown { value: "".to_string(), markdown_type: MarkdownType::Empty, },