Skip to content

[WIP] ✨ Enhance Module System #792

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 6 commits into
base: main
Choose a base branch
from
Open

[WIP] ✨ Enhance Module System #792

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
aca5a2b
✨ Add module token to lexer
harehare Oct 26, 2025
02b2fd3
✨ Add support for parsing module tokens in AST Parser
harehare Oct 27, 2025
88e12ae
✨ Add metadata field to Module struct
harehare Oct 27, 2025
3d1c60a
✨ Update RuntimeValue to include Module variant
harehare Oct 27, 2025
964262f
🔨 Update: Modify debug output for function representation in tests
harehare Oct 28, 2025
9227d73
✨ Add support for Import expression in AST, parser, evaluator, lexer,…
harehare Oct 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion crates/mq-lang/src/ast/node.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ impl Node {
.unwrap_or_else(|| arena[self.token_id].range.end.clone());
Range { start, end }
}
Expr::Paren(node) => node.range(Shared::clone(&arena)),
Expr::Paren(node) | Expr::Module(node) => node.range(Shared::clone(&arena)),
Expr::Try(try_expr, catch_expr) => {
let start = try_expr.range(Shared::clone(&arena)).start;
let end = catch_expr.range(Shared::clone(&arena)).end;
Expand All @@ -116,6 +116,7 @@ impl Node {
| Expr::Ident(_)
| Expr::Selector(_)
| Expr::Include(_)
| Expr::Import(_, _)
| Expr::InterpolatedString(_)
| Expr::Nodes
| Expr::Self_
Expand Down Expand Up @@ -294,6 +295,8 @@ pub enum Expr {
If(Branches),
Match(Shared<Node>, MatchArms),
Include(Literal),
Import(Literal, Option<Literal>),
Module(Shared<Node>),
Self_,
Nodes,
Paren(Shared<Node>),
Expand Down
100 changes: 100 additions & 0 deletions crates/mq-lang/src/ast/parser.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,13 @@ impl<'a, 'alloc> Parser<'a, 'alloc> {
TokenKind::Nodes => {
return Err(ParseError::UnexpectedToken((**token).clone()));
}
TokenKind::Module if root => {
let ast = self.parse_module(Shared::clone(token))?;
asts.push(ast);
}
TokenKind::Module => {
return Err(ParseError::UnexpectedToken((**token).clone()));
}
TokenKind::NewLine | TokenKind::Tab(_) | TokenKind::Whitespace(_) => unreachable!(),
_ => {
let ast = self.parse_expr(Shared::clone(token))?;
Expand Down Expand Up @@ -219,6 +226,7 @@ impl<'a, 'alloc> Parser<'a, 'alloc> {
TokenKind::Match => self.parse_expr_match(Shared::clone(&token)),
TokenKind::InterpolatedString(_) => self.parse_interpolated_string(token),
TokenKind::Include => self.parse_include(token),
TokenKind::Import => self.parse_import(token),
TokenKind::Self_ => self.parse_self(token),
TokenKind::Break => self.parse_break(token),
TokenKind::Continue => self.parse_continue(token),
Expand All @@ -239,6 +247,24 @@ impl<'a, 'alloc> Parser<'a, 'alloc> {
}
}

fn parse_module(&mut self, token: Shared<Token>) -> Result<Shared<Node>, ParseError> {
match &token.kind {
TokenKind::Module => {
let next_token = match self.tokens.next() {
Some(t) => t,
None => return Err(ParseError::UnexpectedEOFDetected(self.module_id)),
};
let metadata = self.parse_dict(Shared::clone(next_token))?;

Ok(Shared::new(Node {
token_id: self.token_arena.alloc(token),
expr: Shared::new(Expr::Module(metadata)),
}))
}
_ => Err(ParseError::UnexpectedToken((*token).clone())),
}
}

fn parse_symbol(&mut self, token: Shared<Token>) -> Result<Shared<Node>, ParseError> {
match &token.kind {
TokenKind::Colon => {
Expand Down Expand Up @@ -1359,6 +1385,39 @@ impl<'a, 'alloc> Parser<'a, 'alloc> {
}
}

#[inline(always)]
fn parse_import(&mut self, import_token: Shared<Token>) -> Result<Shared<Node>, ParseError> {
let token_id = self.token_arena.alloc(import_token);
let token = match self.tokens.peek() {
Some(token) => Ok(Shared::clone(token)),
None => Err(ParseError::UnexpectedEOFDetected(self.module_id)),
}?;

match &token.kind {
TokenKind::StringLiteral(module) => {
self.tokens.next();

match self.tokens.peek() {
Some(token) => match &token.kind {
TokenKind::StringLiteral(module) => Ok(Shared::new(Node {
token_id,
expr: Shared::new(Expr::Import(
Literal::String(module.to_owned()),
None,
)),
})),
_ => Err(ParseError::InsufficientTokens((***token).clone())),
},
None => Ok(Shared::new(Node {
token_id,
expr: Shared::new(Expr::Import(Literal::String(module.to_owned()), None)),
})),
}
}
_ => Err(ParseError::InsufficientTokens((*token).clone())),
}
}

fn parse_interpolated_string(
&mut self,
token: Shared<Token>,
Expand Down Expand Up @@ -5490,6 +5549,47 @@ mod tests {
token(TokenKind::Eof)
],
Err(ParseError::UnexpectedEOFDetected(Module::TOP_LEVEL_MODULE_ID)))]
#[case::module_simple(
vec![
token(TokenKind::Module),
token(TokenKind::LBrace),
token(TokenKind::StringLiteral("key".to_owned())),
token(TokenKind::Colon),
token(TokenKind::StringLiteral("value".to_owned())),
token(TokenKind::RBrace),
token(TokenKind::Eof)
],
Ok(vec![
Shared::new(Node {
token_id: 0.into(),
expr: Shared::new(Expr::Module(
Shared::new(Node {
token_id: 0.into(),
expr: Shared::new(Expr::Call(
IdentWithToken::new_with_token(constants::DICT, Some(Shared::new(token(TokenKind::LBrace)))),
smallvec![
Shared::new(Node {
token_id: 0.into(),
expr: Shared::new(Expr::Call(
IdentWithToken::new_with_token(constants::ARRAY, Some(Shared::new(token(TokenKind::StringLiteral("key".to_owned()))))),
smallvec![
Shared::new(Node {
token_id: 1.into(),
expr: Shared::new(Expr::Literal(Literal::String("key".to_owned()))),
}),
Shared::new(Node {
token_id: 2.into(),
expr: Shared::new(Expr::Literal(Literal::String("value".to_owned()))),
}),
],
)),
}),
],
)),
}),
)),
})
]))]
fn test_parse(#[case] input: Vec<Token>, #[case] expected: Result<Program, ParseError>) {
let mut arena = Arena::new(10);
let tokens: Vec<Shared<Token>> = input.into_iter().map(Shared::new).collect();
Expand Down
107 changes: 104 additions & 3 deletions crates/mq-lang/src/eval.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,9 @@ impl Evaluator {
ast::Expr::Include(module_id) => {
self.eval_include(module_id.to_owned())?;
}
ast::Expr::Import(module_path, alias) => {
self.eval_import(module_path.to_owned(), alias.to_owned())?;
}
_ => nodes.push(Shared::clone(node)),
};

Expand Down Expand Up @@ -219,9 +222,9 @@ impl Evaluator {

Ok(match value {
RuntimeValue::None => child_node.to_fragment(),
RuntimeValue::Function(_, _, _) | RuntimeValue::NativeFunction(_) => {
mq_markdown::Node::Empty
}
RuntimeValue::Function(_, _, _)
| RuntimeValue::NativeFunction(_)
| RuntimeValue::Module(_) => mq_markdown::Node::Empty,
RuntimeValue::Array(_)
| RuntimeValue::Dict(_)
| RuntimeValue::Boolean(_)
Expand Down Expand Up @@ -390,6 +393,97 @@ impl Evaluator {
}
}

fn eval_import(
&mut self,
module_path: ast::Literal,
alias: Option<ast::Literal>,
) -> Result<(), EvalError> {
match module_path {
ast::Literal::String(module_name) => {
let module = self
.module_loader
.load_from_file(&module_name, Shared::clone(&self.token_arena))?;

if let Some(module) = module {
// Create a new environment for the module exports
let module_env = Shared::new(SharedCell::new(Env::with_parent(
Shared::downgrade(&self.env),
)));

// Load module contents into the module environment
for node in &module.modules {
if let ast::Expr::Include(_) = &*node.expr {
self.eval_expr(&RuntimeValue::NONE, node, &Shared::clone(&module_env))?;
}
}

for node in &module.functions {
if let ast::Expr::Def(ident, params, program) = &*node.expr {
define(
&module_env,
ident.name,
RuntimeValue::Function(
params.clone(),
program.clone(),
Shared::clone(&module_env),
),
);
}
}

for node in &module.vars {
if let ast::Expr::Let(ident, node) = &*node.expr {
let val = self.eval_expr(
&RuntimeValue::NONE,
node,
&Shared::clone(&module_env),
)?;
define(&module_env, ident.name, val);
}
}

let module_name_to_use = match alias {
Some(ast::Literal::String(alias_name)) => alias_name,
_ => {
if let Some(metadata) = module.metadata {
match self.eval_expr(
&RuntimeValue::NONE,
&metadata,
&Shared::clone(&module_env),
)? {
RuntimeValue::Dict(dict) => dict
.get(&Ident::new("name"))
.map(|name| name.to_string())
.unwrap_or(module.name),
_ => module.name,
}
} else {
module.name
}
}
};

// Register the module in the environment
let module_runtime_value = RuntimeValue::Module(runtime_value::ModuleEnv::new(
&module_name_to_use,
module_env,
));

define(
&self.env,
Ident::new(&module_name_to_use),
module_runtime_value,
);
}

Ok(())
}
_ => Err(EvalError::ModuleLoadError(
module::ModuleError::InvalidModule,
)),
}
}

#[inline(always)]
fn eval_selector_expr(runtime_value: &RuntimeValue, ident: &ast::Selector) -> RuntimeValue {
match runtime_value {
Expand Down Expand Up @@ -577,6 +671,13 @@ impl Evaluator {
self.eval_include(module_id.to_owned())?;
Ok(runtime_value.clone())
}
ast::Expr::Import(module_path, alias) => {
self.eval_import(module_path.to_owned(), alias.to_owned())?;
Ok(runtime_value.clone())
}
ast::Expr::Module(_module) => {
todo!()
}
ast::Expr::Match(value_node, arms) => {
self.eval_match(runtime_value, value_node, arms, env)
}
Expand Down
9 changes: 9 additions & 0 deletions crates/mq-lang/src/eval/env.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,11 @@ impl Variable {
value: "native function".to_string(),
type_field: "native_function".to_string(),
},
RuntimeValue::Module(m) => Variable {
name: m.name().to_string(),
value: format!("module/{}", m.len()),
type_field: "module".to_string(),
},
RuntimeValue::None => Variable {
name: ident.to_string(),
value: "None".to_string(),
Expand Down Expand Up @@ -129,6 +134,10 @@ impl Env {
}
}

pub fn len(&self) -> usize {
self.context.len()
}

#[inline(always)]
pub fn define(&mut self, ident: Ident, runtime_value: RuntimeValue) {
self.context.insert(ident, runtime_value);
Expand Down
17 changes: 14 additions & 3 deletions crates/mq-lang/src/eval/module.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ pub struct Module {
pub functions: Program,
pub modules: Program,
pub vars: Program,
pub metadata: Option<Shared<ast::Node>>,
}

impl Module {
Expand Down Expand Up @@ -156,6 +157,11 @@ impl ModuleLoader {
.cloned()
.collect::<Vec<_>>();

let metadata = program
.iter()
.find(|node| matches!(*node.expr, ast::Expr::Module(_)))
.cloned();

if program.len() != functions.len() + modules.len() + vars.len() {
return Err(ModuleError::InvalidModule);
}
Expand All @@ -165,6 +171,7 @@ impl ModuleLoader {
functions,
modules,
vars,
metadata,
}))
}

Expand Down Expand Up @@ -338,7 +345,8 @@ mod tests {
module_id: 1.into()
}))),
Shared::new(ast::Node{token_id: 2.into(), expr: Shared::new(ast::Expr::Literal(ast::Literal::String("value".to_string())))})
))})]
))})],
metadata: None
})))]
#[case::load3("def test(): 1;".to_string(), Ok(Some(Module{
name: "test".to_string(),
Expand All @@ -355,7 +363,8 @@ mod tests {
Shared::new(ast::Node{token_id: 2.into(), expr: Shared::new(ast::Expr::Literal(ast::Literal::Number(1.into())))})
]
))})],
vars: Vec::new()
vars: Vec::new(),
metadata: None
})))]
#[case::load4("def test(a, b): add(a, b);".to_string(), Ok(Some(Module{
name: "test".to_string(),
Expand Down Expand Up @@ -388,7 +397,8 @@ mod tests {
],
))})]
))})],
vars: Vec::new()
vars: Vec::new(),
metadata: None
})))]
fn test_load(
token_arena: Shared<SharedCell<crate::arena::Arena<Shared<Token>>>>,
Expand All @@ -407,6 +417,7 @@ mod tests {
functions: Vec::new(),
modules: Vec::new(), // Assuming the csv.mq only contains definitions or is empty for this test
vars: Vec::new(),
metadata: None
})))]
fn test_load_standard_module(
token_arena: Shared<SharedCell<crate::arena::Arena<Shared<Token>>>>,
Expand Down
Loading