From 5c2b93a635dde66ed35324e3582e017689186942 Mon Sep 17 00:00:00 2001 From: Zffu <103074097+Zffu@users.noreply.github.com> Date: Sun, 29 Mar 2026 13:07:04 +0200 Subject: [PATCH 1/2] feat: added new tokens --- compiler/lexer/src/token.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/compiler/lexer/src/token.rs b/compiler/lexer/src/token.rs index f58a2073..38154340 100644 --- a/compiler/lexer/src/token.rs +++ b/compiler/lexer/src/token.rs @@ -15,6 +15,12 @@ pub enum LexerTokenType { Function, ShadowFunction, + /// A comment on a function + LocalComment(String), + + /// A comment on a whole file + GlobalComment(String), + Var, Struct, Layout, From d756aa55a78e86eab5caa24f21957f57c3406989 Mon Sep 17 00:00:00 2001 From: Zffu <103074097+Zffu@users.noreply.github.com> Date: Sun, 29 Mar 2026 14:04:46 +0200 Subject: [PATCH 2/2] feat: added comments in lexer --- compiler/lexer/src/lexer.rs | 63 +++++++++++++++++++++++++++++++++++-- compiler/lexer/src/token.rs | 5 +-- 2 files changed, 61 insertions(+), 7 deletions(-) diff --git a/compiler/lexer/src/lexer.rs b/compiler/lexer/src/lexer.rs index 8f6ca56a..0117eb88 100644 --- a/compiler/lexer/src/lexer.rs +++ b/compiler/lexer/src/lexer.rs @@ -43,11 +43,9 @@ pub fn lexer_parse_file(file_path: &String) -> CompilerResult> { let mut tokens: Vec = Vec::new(); let mut i: usize = 0; - let mut line: usize = 1; - let mut last_line_break: usize = 0; - + while i < contents.len() { let c: char = contents.chars().nth(i).unwrap(); @@ -99,6 +97,21 @@ pub fn lexer_parse_file(file_path: &String) -> CompilerResult> { i -= 2; // Try parsing operator as normal token. } + if c == '/' { + let cc = contents.chars().nth(i + 1).unwrap(); + if cc == '/' { + let col = i - last_line_break; + + tokens.push(parse_comment(&contents, &mut i, Position::new(file_path.to_string(), line, col))?); + continue; + } else if cc == '.' { + let col = i - last_line_break; + + tokens.push(parse_global_comment(&contents, &mut i, Position::new(file_path.to_string(), line, col))?); + continue; + } + } + i += 1; @@ -131,6 +144,50 @@ pub fn lexer_parse_file(file_path: &String) -> CompilerResult> { Ok(tokens) } +fn parse_comment(contents: &String, ind: &mut usize, start_pos: Position) -> CompilerResult { + *ind += 2; + + let start = *ind; + let mut end = start; + + for (i, c) in contents[start..].char_indices() { + if c == '\n' || c == '\0' { + end = start + i + c.len_utf8(); + break; + } + + end = start + i + c.len_utf16(); + } + + let slice = &contents[*ind + 1..end - 1]; + + *ind = end; + + return Ok(LexerToken::new(start_pos, end - start, LexerTokenType::Comment(slice.to_string()))) +} + +fn parse_global_comment(contents: &String, ind: &mut usize, start_pos: Position) -> CompilerResult { + *ind += 2; + + let start = *ind; + let mut end = start; + + for (i, c) in contents[start..].char_indices() { + if c == '\n' || c == '\0' { + end = start + i + c.len_utf8(); + break; + } + + end = start + i + c.len_utf16(); + } + + let slice = &contents[*ind + 1..end - 1]; + + *ind = end; + + return Ok(LexerToken::new(start_pos, end - start, LexerTokenType::GlobalComment(slice.to_string()))) +} + fn parse_math_operator(contents: &String, ind: &mut usize, start_pos: Position) -> CompilerResult { let operator_char = contents.chars().nth(*ind).unwrap(); diff --git a/compiler/lexer/src/token.rs b/compiler/lexer/src/token.rs index 38154340..72261ba1 100644 --- a/compiler/lexer/src/token.rs +++ b/compiler/lexer/src/token.rs @@ -15,10 +15,7 @@ pub enum LexerTokenType { Function, ShadowFunction, - /// A comment on a function - LocalComment(String), - - /// A comment on a whole file + Comment(String), GlobalComment(String), Var,