From d04256d7edd7dfa0db816621cf5d5bbc59785a82 Mon Sep 17 00:00:00 2001
From: Richard Carson <r.carson@calian.com>
Date: Fri, 27 Oct 2023 15:21:35 -0400
Subject: [PATCH] fix parsing comments

---
 src/grammar.pest    | 10 +++++-----
 src/handlers/mod.rs | 16 +++++++++++++---
 src/token.rs        |  5 +++++
 3 files changed, 23 insertions(+), 8 deletions(-)

diff --git a/src/grammar.pest b/src/grammar.pest
index 3a3f974..8002afa 100644
--- a/src/grammar.pest
+++ b/src/grammar.pest
@@ -77,10 +77,8 @@ term = {lparen ~ toplevel_expression ~ rparen | atomic_value}
 
 assignment_expression = {(assignment_prefix | index_assignment_prefix) ~ toplevel_expression}
 expression = {function_assignment|assignment_expression|toplevel_expression}
-comment = _{"//" ~ (!eol ~ ANY)*}
-eol = @{NEWLINE+|";"|EOI}
-line = {(comment | (expression ~ (decorator ~ identifier)? ~ comment?)) ~ eol}
-script = {eol? ~ line+ | eol}
+line = {WHITESPACE* ~ eol | WHITESPACE* ~  ((expression ~ (decorator ~ identifier)?)) ~ eol?}
+script = {line* ~ EOI}
 
 // Value errors
 errors = {
@@ -100,4 +98,6 @@ error_unterminated_paren = @{lparen ~ ANY*}
 error_unexpected_decorator = @{decorator ~ identifier ~ WHITESPACE* ~ (!WHITESPACE~!eol~ANY)+ | SOI ~ decorator ~ ANY*}
 error_unexpected_postfix =  @{(factorial ~ !"=") ~ ANY*}
 
-WHITESPACE = _{" " | "\t" | "\\\n"}
\ No newline at end of file
+WHITESPACE = _{" " | "\t" | "\\\n" | comment}
+comment = _{"//" ~ (!eol ~ ANY)* | "/*" ~ (!eol ~ !"*/" ~ ANY)* ~ "*/"}
+eol = @{NEWLINE|";"}
\ No newline at end of file
diff --git a/src/handlers/mod.rs b/src/handlers/mod.rs
index b556b00..b62b470 100644
--- a/src/handlers/mod.rs
+++ b/src/handlers/mod.rs
@@ -109,6 +109,7 @@ impl LavendeuxHandler for Handler {
 type RuleHandler = fn(token: &mut Token, state: &mut ParserState) -> Option<Error>;
 fn handler_table() -> HashMap<Rule, RuleHandler> {
     HashMap::from([
+        (Rule::EOI, rule_eoi as RuleHandler),
         (Rule::script, rule_script as RuleHandler),
         (Rule::line, rule_line as RuleHandler),
         (Rule::term, rule_term as RuleHandler),
@@ -127,16 +128,25 @@ fn handler_table() -> HashMap<Rule, RuleHandler> {
     .collect()
 }
 
+fn rule_eoi(token: &mut Token, _state: &mut ParserState) -> Option<Error> {
+    token.set_text("");
+    None
+}
+
 /// A series of lines
 fn rule_script(token: &mut Token, _state: &mut ParserState) -> Option<Error> {
     // Concatenate output from all child tokens (lines)
+    println!("{:?}", token);
+
     token.set_text(
         &token
             .children()
             .iter()
             .map(|t| {
                 t.text().to_string()
-                    + if !t.children().is_empty() {
+                    + if !t.children().is_empty()
+                        && t.children().last().unwrap().rule() == Rule::eol
+                    {
                         t.children().last().unwrap().text()
                     } else {
                         ""
@@ -146,8 +156,8 @@ fn rule_script(token: &mut Token, _state: &mut ParserState) -> Option<Error> {
             .join(""),
     );
 
-    // Set script value if there is only one line
-    if token.children().len() == 1 {
+    // Set script value
+    if token.children().len() > 0 {
         token.set_value(token.child(0).unwrap().value());
     }
 
diff --git a/src/token.rs b/src/token.rs
index 51bc298..ca336fc 100644
--- a/src/token.rs
+++ b/src/token.rs
@@ -432,6 +432,11 @@ mod test_token {
         assert_token_value!("false ? 1 : 2", Value::Integer(2));
         assert_token_value!("false ? 1/0 : 2", Value::Integer(2));
 
+        // Comments
+        assert_token_value!("/* test */ true", Value::from(true));
+        assert_token_value!("/* test */ true // test", Value::from(true));
+        assert_token_text!("/* test */ 15 @hex// test", "0xf");
+
         // arrays
         assert_token_value!(
             "[10, 12] + [1.2, 1.3]",