Added support for #[logos(skip "...")] (#284)

* Tidy up #[logos(...)] attr parsing

* Don't disregard syn parse errors

* Added support for #[logos(skip "...")]

* Minor fmt

README.md

@@ -28,6 +28,7 @@ To achieve those, **Logos**:
  use logos::Logos;
 
  #[derive(Logos, Debug, PartialEq)]
+ #[logos(skip r"[ \t\n\f]+")] // Ignore this regex pattern between tokens
  enum Token {
      // Tokens can be literal strings, of any length.
      #[token("fast")]
@@ -39,12 +40,6 @@ To achieve those, **Logos**:
      // Or regular expressions.
      #[regex("[a-zA-Z]+")]
      Text,
-
-     // Logos requires one token variant to define whitespace,
-     // or any other matches we wish to skip.
-     // It can be named anything you wish.
-     #[regex(r"[ \t\n\f]+", logos::skip)]
-     Ignored,
  }
 
  fn main() {
@@ -96,10 +91,8 @@ which can be used to put data into a variant:
  }
 
  #[derive(Logos, Debug, PartialEq)]
+ #[logos(skip r"[ \t\n\f]+")]
  enum Token {
-     #[regex(r"[ \t\n\f]+", logos::skip)]
-     Ignored,
-
      // Callbacks can use closure syntax, or refer
      // to a function defined elsewhere.
      //

logos-codegen/src/generator/leaf.rs

@@ -39,6 +39,14 @@ impl<'a> Generator<'a> {
                     callback(lex).construct(#constructor, lex);
                 }
             }
+            Some(Callback::Skip(_)) => {
+                quote! {
+                    #bump
+
+                    lex.trivia();
+                    #name::lex(lex);
+                }
+            }
             None if matches!(leaf.field, MaybeVoid::Void) => quote! {
                 #bump
                 lex.set(Ok(#name::#ident));

logos-codegen/src/leaf.rs

@@ -1,5 +1,5 @@
 use std::cmp::{Ord, Ordering};
-use std::fmt::{self, Debug};
+use std::fmt::{self, Debug, Display};
 
 use proc_macro2::{Span, TokenStream};
 use syn::{spanned::Spanned, Ident};
@@ -9,7 +9,7 @@ use crate::util::MaybeVoid;
 
 #[derive(Clone)]
 pub struct Leaf<'t> {
-    pub ident: &'t Ident,
+    pub ident: Option<&'t Ident>,
     pub span: Span,
     pub priority: usize,
     pub field: MaybeVoid,
@@ -20,6 +20,7 @@ pub struct Leaf<'t> {
 pub enum Callback {
     Label(TokenStream),
     Inline(Box<InlineCallback>),
+    Skip(Span),
 }
 
 #[derive(Clone)]
@@ -40,21 +41,32 @@ impl Callback {
         match self {
             Callback::Label(tokens) => tokens.span(),
             Callback::Inline(inline) => inline.span,
+            Callback::Skip(span) => *span,
         }
     }
 }
 
 impl<'t> Leaf<'t> {
     pub fn new(ident: &'t Ident, span: Span) -> Self {
         Leaf {
-            ident,
+            ident: Some(ident),
             span,
             priority: 0,
             field: MaybeVoid::Void,
             callback: None,
         }
     }
 
+    pub fn new_skip(span: Span) -> Self {
+        Leaf {
+            ident: None,
+            span,
+            priority: 0,
+            field: MaybeVoid::Void,
+            callback: Some(Callback::Skip(span)),
+        }
+    }
+
     pub fn callback(mut self, callback: Option<Callback>) -> Self {
         self.callback = callback;
         self
@@ -85,12 +97,22 @@ impl<'t> From<Leaf<'t>> for Node<Leaf<'t>> {
 
 impl Debug for Leaf<'_> {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "::{}", self.ident)?;
+        write!(f, "::{}", self)?;
 
         match self.callback {
             Some(Callback::Label(ref label)) => write!(f, " ({})", label),
             Some(Callback::Inline(_)) => f.write_str(" (<inline>)"),
+            Some(Callback::Skip(_)) => f.write_str(" (<skip>)"),
             None => Ok(()),
         }
     }
 }
+
+impl Display for Leaf<'_> {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self.ident {
+            Some(ident) => Display::fmt(ident, f),
+            None => f.write_str("<skip>"),
+        }
+    }
+}

logos-codegen/src/lib.rs

@@ -19,7 +19,7 @@ mod util;
 use generator::Generator;
 use graph::{DisambiguationError, Fork, Graph, Rope};
 use leaf::Leaf;
-use parser::{Mode, Parser};
+use parser::{IgnoreFlags, Mode, Parser};
 use quote::ToTokens;
 use util::MaybeVoid;
 
@@ -30,9 +30,7 @@ use syn::spanned::Spanned;
 use syn::{Fields, ItemEnum};
 
 const LOGOS_ATTR: &str = "logos";
-const EXTRAS_ATTR: &str = "extras";
 const ERROR_ATTR: &str = "error";
-const END_ATTR: &str = "end";
 const TOKEN_ATTR: &str = "token";
 const REGEX_ATTR: &str = "regex";
 
@@ -50,25 +48,30 @@ pub fn generate(input: TokenStream) -> TokenStream {
 
     for attr in &mut item.attrs {
         parser.try_parse_logos(attr);
-
-        // TODO: Remove in future versions
-        if attr.path.is_ident(EXTRAS_ATTR) {
-            parser.err(
-                "\
-                #[extras] attribute is deprecated. Use #[logos(extras = Type)] instead.\n\
-                \n\
-                For help with migration see release notes: \
-                https://github.com/maciejhirsz/logos/releases\
-                ",
-                attr.span(),
-            );
-        }
     }
 
     let mut ropes = Vec::new();
     let mut regex_ids = Vec::new();
     let mut graph = Graph::new();
 
+    {
+        let errors = &mut parser.errors;
+
+        for literal in &parser.skips {
+            match literal.to_mir(&parser.subpatterns, IgnoreFlags::Empty, errors) {
+                Ok(mir) => {
+                    let then = graph.push(Leaf::new_skip(literal.span()).priority(mir.priority()));
+                    let id = graph.regex(mir, then);
+
+                    regex_ids.push(id);
+                }
+                Err(err) => {
+                    errors.err(err, literal.span());
+                }
+            }
+        }
+    }
+
     for variant in &mut item.variants {
         let field = match &mut variant.fields {
             Fields::Unit => MaybeVoid::Void,
@@ -122,18 +125,6 @@ pub fn generate(input: TokenStream) -> TokenStream {
                         attr.span(),
                     );
                 }
-                END_ATTR => {
-                    // TODO: Remove in future versions
-                    parser.err(
-                        "\
-                        Since 0.11 Logos no longer requires the #[end] variant.\n\
-                        \n\
-                        For help with migration see release notes: \
-                        https://github.com/maciejhirsz/logos/releases\
-                        ",
-                        attr.span(),
-                    );
-                }
                 TOKEN_ATTR => {
                     let definition = match parser.parse_definition(attr) {
                         Some(definition) => definition,
@@ -225,7 +216,7 @@ pub fn generate(input: TokenStream) -> TokenStream {
 
     let impl_logos = |body| {
         quote! {
-            impl<'s> ::logos::Logos<'s> for #this {
+            impl<'s> #logos_path::Logos<'s> for #this {
                 type Error = #error_type;
 
                 type Extras = #extras;
@@ -266,14 +257,11 @@ pub fn generate(input: TokenStream) -> TokenStream {
             parser.err(
                 format!(
                     "\
-                    A definition of variant `{0}` can match the same input as another definition of variant `{1}`.\n\
+                    A definition of variant `{a}` can match the same input as another definition of variant `{b}`.\n\
                     \n\
                     hint: Consider giving one definition a higher priority: \
-                    #[regex(..., priority = {2})]\
+                    #[regex(..., priority = {disambiguate})]\
                     ",
-                    a.ident,
-                    b.ident,
-                    disambiguate,
                 ),
                 a.span
             );
@@ -354,8 +342,6 @@ fn strip_attrs_from_vec(attrs: &mut Vec<syn::Attribute>) {
 
 fn is_logos_attr(attr: &syn::Attribute) -> bool {
     attr.path.is_ident(LOGOS_ATTR)
-        || attr.path.is_ident(EXTRAS_ATTR)
-        || attr.path.is_ident(END_ATTR)
         || attr.path.is_ident(TOKEN_ATTR)
         || attr.path.is_ident(REGEX_ATTR)
 }