Add FilterResult type (#236)

maciejhirsz · Jun 8, 2022 · 89d9a8b · 89d9a8b
1 parent 9913810
commit 89d9a8b
Show file tree

Hide file tree

Showing 3 changed files with 84 additions and 11 deletions.
diff --git a/README.md b/README.md
@@ -131,16 +131,17 @@ fn main() {
 
 Logos can handle callbacks with following return types:
 
-| Return type                       | Produces                                           |
-|-----------------------------------|----------------------------------------------------|
-| `()`                              | `Token::Unit`                                      |
-| `bool`                            | `Token::Unit` **or** `<Token as Logos>::ERROR`     |
-| `Result<(), _>`                   | `Token::Unit` **or** `<Token as Logos>::ERROR`     |
-| `T`                               | `Token::Value(T)`                                  |
-| `Option<T>`                       | `Token::Value(T)` **or** `<Token as Logos>::ERROR` |
-| `Result<T, _>`                    | `Token::Value(T)` **or** `<Token as Logos>::ERROR` |
-| `Skip`                            | _skips matched input_                              |
-| `Filter<T>`                       | `Token::Value(T)` **or** _skips matched input_     |
+| Return type                       | Produces                                                                         |
+|-----------------------------------|----------------------------------------------------------------------------------|
+| `()`                              | `Token::Unit`                                                                    |
+| `bool`                            | `Token::Unit` **or** `<Token as Logos>::ERROR`                                   |
+| `Result<(), _>`                   | `Token::Unit` **or** `<Token as Logos>::ERROR`                                   |
+| `T`                               | `Token::Value(T)`                                                                |
+| `Option<T>`                       | `Token::Value(T)` **or** `<Token as Logos>::ERROR`                               |
+| `Result<T, _>`                    | `Token::Value(T)` **or** `<Token as Logos>::ERROR`                               |
+| `Skip`                            | _skips matched input_                                                            |
+| `Filter<T>`                       | `Token::Value(T)` **or** _skips matched input_                                   |
+| `FilterResult<T>`                 | `Token::Value(T)` **or** `<Token as Logos>::ERROR>` **or** _skips matched input_ |
 
 Callbacks can be also used to do perform more specialized lexing in place
 where regular expressions are too limiting. For specifics look at

diff --git a/logos/src/internal.rs b/logos/src/internal.rs
@@ -1,5 +1,5 @@
 use crate::source::Chunk;
-use crate::{Filter, Lexer, Logos, Skip};
+use crate::{Filter, FilterResult, Lexer, Logos, Skip};
 
 /// Trait used by the functions contained in the `Lexicon`.
 ///
@@ -121,3 +121,19 @@ impl<'s, P, T: Logos<'s>> CallbackResult<'s, P, T> for Filter<P> {
         }
     }
 }
+
+impl<'s, P, T: Logos<'s>> CallbackResult<'s, P, T> for FilterResult<P> {
+    fn construct<Constructor>(self, c: Constructor, lex: &mut Lexer<'s, T>)
+    where
+        Constructor: Fn(P) -> T,
+    {
+        match self {
+            FilterResult::Emit(product) => lex.set(c(product)),
+            FilterResult::Skip => {
+                lex.trivia();
+                T::lex(lex);
+            }
+            FilterResult::Error => lex.set(T::ERROR),
+        }
+    }
+}
diff --git a/logos/src/lib.rs b/logos/src/lib.rs
@@ -297,6 +297,62 @@ pub enum Filter<T> {
     Skip,
 }
 
+/// Type that can be returned from a callback, either producing a field
+/// for a token, skipping it, or emitting an error.
+/// 
+/// # Example
+/// 
+/// ```rust
+/// use logos::{Logos, FilterResult};
+///
+/// #[derive(Logos, Debug, PartialEq)]
+/// enum Token {
+///     #[regex(r"[ \n\f\t]+", logos::skip)]
+///     #[error]
+///     Error,
+///
+///     #[regex("[0-9]+", |lex| {
+///         let n: u64 = lex.slice().parse().unwrap();
+///
+///         // Only emit a token if `n` is an even number.
+///         if n % 2 == 0 {
+///             // Emit an error if `n` is 10.
+///             if n == 10 {
+///                 FilterResult::Error
+///             } else {
+///                 FilterResult::Emit(n)
+///             }
+///         } else {
+///             FilterResult::Skip
+///         }
+///     })]
+///     NiceEvenNumber(u64)
+/// }
+///
+/// let tokens: Vec<_> = Token::lexer("20 11 42 23 100 10").collect();
+///
+/// assert_eq!(
+///     tokens,
+///     &[
+///         Token::NiceEvenNumber(20),
+///         // skipping 11
+///         Token::NiceEvenNumber(42),
+///         // skipping 23
+///         Token::NiceEvenNumber(100),
+///         // error at 10
+///         Token::Error,
+///     ]
+/// );
+/// ```
+pub enum FilterResult<T> {
+    /// Emit a token with a given value `T`. Use `()` for unit variants without fields.
+    Emit(T),
+    /// Skip current match, analog to [`Skip`](./struct.Skip.html).
+    Skip,
+    /// Emit a `<Token as Logos>::ERROR` token.
+    Error,
+}
+
 /// Predefined callback that will inform the `Lexer` to skip a definition.
 ///
 /// # Example