Allow returning errors from lexer

pango-lexer/src/lexer/input.rs

@@ -31,17 +31,21 @@ impl Default for InputIterToken {
 
 impl InputIterToken {
     /// Creates a new [`InputIterToken`].
-    pub(self) fn new(token_start: usize) -> Self {
+    fn new(token_start: usize) -> Self {
         Self {
             source: String::new(),
             pos: (token_start, token_start),
         }
     }
 
     /// Updates the end position of the token.
-    pub(self) fn update_pos_end(&mut self, new_end: usize) {
+    fn update_pos_end(&mut self, new_end: usize) {
         self.pos.1 = new_end;
     }
+
+    pub(super) fn is_empty(&self) -> bool {
+        self.source.is_empty()
+    }
 }
 
 impl Iterator for InputIter<'_> {

pango-lexer/src/lexer/mod.rs

@@ -1,11 +1,15 @@
-use self::input::{InputIter, InputIterToken};
+use self::{
+    error::{LexError, LexResult},
+    input::{InputIter, InputIterToken},
+};
 use crate::{
     fsm::{NDSimulate, Nfa, NfaCompiler, Simulatable, Simulate, StateId},
     regex::{self, parser::error::ParseResult},
 };
 
-use std::collections::BTreeMap;
+use std::{collections::BTreeMap, iter::FusedIterator};
 
+mod error;
 mod input;
 
 /// Finite-state machine based lexer.
@@ -63,7 +67,7 @@ where
     for<'a> Fsm: Simulatable + 'a,
     for<'a> Fsm::Simulator<'a>: NDSimulate,
 {
-    type Item = Token<TokenKind>;
+    type Item = LexResult<Token<TokenKind>>;
 
     fn next(&mut self) -> Option<Self::Item> {
         let mut sim = self.fsm.to_simulator();
@@ -73,8 +77,16 @@ where
             if !sim.can_feed(ch) {
                 // the current token is the longest token we can 'munch'
 
-                return token_kind.map(|token_kind| {
-                    Token::from_input_iter_token(self.iter.consume_token(), token_kind)
+                return Some(match token_kind {
+                    Some(token_kind) => Ok(Token::from_input_iter_token(
+                        self.iter.consume_token(),
+                        token_kind,
+                    )),
+                    None => {
+                        // tokenizer could not identify the token
+                        let token = self.iter.get_token();
+                        Err(LexError(token.pos.0, token.pos.1))
+                    }
                 });
             }
 
@@ -85,7 +97,7 @@ where
 
                 // Get the first final state the Simulator is in.
                 //
-                // Note: this assumes that the lower final state ids, have the highest
+                // NOTE: this assumes that the lower final state ids, have the highest
                 // precedence. This is enforced by the order of expression
                 // compilation.
                 let final_state = sim
@@ -110,16 +122,35 @@ where
             }
         }
 
-        if sim.is_accepting() {
+        if dbg!(sim.is_accepting()) {
             token_kind.map(|token_kind| {
-                Token::from_input_iter_token(self.iter.consume_token(), token_kind)
+                Ok(Token::from_input_iter_token(
+                    self.iter.consume_token(),
+                    token_kind,
+                ))
             })
         } else {
-            None
+            // check if characters are remaining
+            self.iter.accept_suffix();
+            let token = self.iter.get_token();
+
+            if token.is_empty() {
+                None
+            } else {
+                let token = self.iter.consume_token();
+                Some(Err(LexError(token.pos.0, token.pos.1)))
+            }
         }
     }
 }
 
+impl<TokenKind, Fsm> FusedIterator for Lexer<'_, TokenKind, Fsm>
+where
+    for<'a> Fsm: Simulatable + 'a,
+    for<'a> Fsm::Simulator<'a>: NDSimulate,
+{
+}
+
 impl<TokenKind> Lexer<'_, TokenKind, Nfa> {
     /// Creates a `LexerGenerator`.
     pub fn builder() -> LexerGenerator<TokenKind> {
@@ -247,7 +278,7 @@ mod tests {
     macro_rules! assert_eq_tokens {
         ($lhs:expr, $rhs:expr) => {
             for (expected, actual) in $lhs.into_iter().zip($rhs) {
-                assert_eq!(expected, actual);
+                assert_eq!(Ok(expected), actual);
             }
         };
     }
@@ -266,22 +297,24 @@ mod tests {
         };
     }
 
+    use crate::lexer::error::LexError;
+
     use super::{Lexer, Token};
 
+    #[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
+    enum Foo {
+        #[default]
+        A,
+        B,
+        C,
+        D {
+            len: usize,
+        },
+        E,
+    }
+
     #[test]
     fn lexer() -> Result<(), Box<dyn std::error::Error>> {
-        #[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
-        enum Foo {
-            #[default]
-            A,
-            B,
-            C,
-            D {
-                len: usize,
-            },
-            E,
-        }
-
         let tokens = Lexer::builder()
             .with_token_unit("aaaa", Foo::A)?
             .with_token_unit("b{4,}", Foo::B)?
@@ -308,4 +341,15 @@ mod tests {
 
         Ok(())
     }
+
+    #[test]
+    fn token_error() -> Result<(), Box<dyn std::error::Error>> {
+        let tokens = Lexer::builder()
+            .with_token_unit("aa", Foo::A)?
+            .tokenize("aaa");
+
+        assert_eq!(tokens.last(), Some(Err(LexError(2, 3))));
+
+        Ok(())
+    }
 }

pango-parser/src/parser/slr.rs

@@ -55,7 +55,12 @@ where
             state: current_state,
         }]);
 
-        while let Some(token) = next_token.or_else(|| input.next()) {
+        while let Some(token) = next_token.map(Ok).or_else(|| input.next()) {
+            let token = match token {
+                Ok(token) => token,
+                Err(_) => return Err(()),
+            };
+
             next_token = Some(token);
             let (parse_node, accept) =
                 self.handle_token(&mut next_token, &mut stack, current_state)?;