Addendum for fn parsing

Changes: * Unit tests for mut parameters and lifetimes * Declaration order of function (dependencies above dependents) * Capture all tokens ('fn' keyword, ';' at end)
PoignardAzur · Jul 11, 2022 · 6d104f7 · 6d104f7
1 parent be136e5
commit 6d104f7
Show file tree

Hide file tree

Showing 28 changed files with 387 additions and 80 deletions.
diff --git a/src/parse_fn.rs b/src/parse_fn.rs
@@ -13,55 +13,6 @@ use std::iter::Peekable;
 
 type TokenIter = Peekable<proc_macro2::token_stream::IntoIter>;
 
-pub(crate) fn consume_fn(
-    tokens: &mut TokenIter,
-    attributes: Vec<Attribute>,
-    vis_marker: Option<VisMarker>,
-) -> Function {
-    let qualifiers = consume_fn_qualifiers(tokens);
-
-    // fn keyword
-    tokens.next().unwrap();
-
-    let fn_name = consume_declaration_name(tokens);
-    let generic_params = consume_generic_params(tokens);
-
-    let (params, tk_params_parens) = match tokens.next().unwrap() {
-        TokenTree::Group(group) if group.delimiter() == Delimiter::Parenthesis => {
-            (parse_fn_params(group.stream()), GroupSpan::new(&group))
-        }
-        _ => panic!("cannot parse function"),
-    };
-
-    let (tk_return_arrow, return_ty) = if let Some((arrow, ty)) = consume_fn_return(tokens) {
-        (Some(arrow), Some(ty))
-    } else {
-        (None, None)
-    };
-
-    let where_clause = consume_where_clause(tokens);
-
-    let function_body = match &tokens.next().unwrap() {
-        TokenTree::Group(group) if group.delimiter() == Delimiter::Brace => Some(group.clone()),
-        TokenTree::Punct(punct) if punct.as_char() == ';' => None,
-        _ => panic!("cannot parse function"),
-    };
-
-    Function {
-        attributes,
-        vis_marker,
-        qualifiers,
-        name: fn_name,
-        generic_params,
-        tk_params_parens,
-        params,
-        where_clause,
-        tk_return_arrow,
-        return_ty,
-        body: function_body,
-    }
-}
-
 pub(crate) fn consume_fn_qualifiers(tokens: &mut TokenIter) -> FunctionQualifiers {
     let tk_default = match tokens.peek() {
         Some(TokenTree::Ident(ident)) if ident == "default" => {
@@ -129,35 +80,6 @@ pub(crate) fn consume_fn_qualifiers(tokens: &mut TokenIter) -> FunctionQualifier
     }
 }
 
-pub(crate) fn consume_fn_return(tokens: &mut TokenIter) -> Option<([Punct; 2], TyExpr)> {
-    let dash = match tokens.peek() {
-        Some(TokenTree::Punct(punct)) if punct.as_char() == '-' => punct.clone(),
-        _ => return None,
-    };
-    tokens.next().unwrap();
-
-    let tip = match tokens.next() {
-        Some(TokenTree::Punct(punct)) if punct.as_char() == '>' => punct,
-        _ => panic!("cannot parse fn return: expected '>' after '-' token"),
-    };
-
-    Some((
-        [dash, tip],
-        TyExpr {
-            tokens: (consume_stuff_until(
-                tokens,
-                |token| match token {
-                    TokenTree::Group(group) if group.delimiter() == Delimiter::Brace => true,
-                    TokenTree::Ident(i) if i == &Ident::new("where", i.span()) => true,
-                    TokenTree::Punct(punct) if punct.as_char() == ';' => true,
-                    _ => false,
-                },
-                true,
-            )),
-        },
-    ))
-}
-
 pub(crate) fn parse_fn_params(tokens: TokenStream) -> Punctuated<FunctionParameter> {
     let mut fields = Punctuated::new();
 
@@ -224,3 +146,89 @@ pub(crate) fn parse_fn_params(tokens: TokenStream) -> Punctuated<FunctionParamet
 
     fields
 }
+
+pub(crate) fn consume_fn_return(tokens: &mut TokenIter) -> Option<([Punct; 2], TyExpr)> {
+    let dash = match tokens.peek() {
+        Some(TokenTree::Punct(punct)) if punct.as_char() == '-' => punct.clone(),
+        _ => return None,
+    };
+    tokens.next().unwrap();
+
+    let tip = match tokens.next() {
+        Some(TokenTree::Punct(punct)) if punct.as_char() == '>' => punct,
+        _ => panic!("cannot parse fn return: expected '>' after '-' token"),
+    };
+
+    Some((
+        [dash, tip],
+        TyExpr {
+            tokens: (consume_stuff_until(
+                tokens,
+                |token| match token {
+                    TokenTree::Group(group) if group.delimiter() == Delimiter::Brace => true,
+                    TokenTree::Ident(i) if i == &Ident::new("where", i.span()) => true,
+                    TokenTree::Punct(punct) if punct.as_char() == ';' => true,
+                    _ => false,
+                },
+                true,
+            )),
+        },
+    ))
+}
+
+pub(crate) fn consume_fn(
+    tokens: &mut TokenIter,
+    attributes: Vec<Attribute>,
+    vis_marker: Option<VisMarker>,
+) -> Function {
+    let qualifiers = consume_fn_qualifiers(tokens);
+
+    // fn keyword
+    let tk_fn_keyword = if let Some(TokenTree::Ident(fn_keyword)) = tokens.next() {
+        fn_keyword
+    } else {
+        panic!("expected 'fn' keyword")
+    };
+
+    let fn_name = consume_declaration_name(tokens);
+    let generic_params = consume_generic_params(tokens);
+
+    let (params, tk_params_parens) = match tokens.next().unwrap() {
+        TokenTree::Group(group) if group.delimiter() == Delimiter::Parenthesis => {
+            (parse_fn_params(group.stream()), GroupSpan::new(&group))
+        }
+        _ => panic!("cannot parse function"),
+    };
+
+    let (tk_return_arrow, return_ty) = if let Some((arrow, ty)) = consume_fn_return(tokens) {
+        (Some(arrow), Some(ty))
+    } else {
+        (None, None)
+    };
+
+    let where_clause = consume_where_clause(tokens);
+
+    let (function_body, tk_semicolon) = match &tokens.next().unwrap() {
+        TokenTree::Group(group) if group.delimiter() == Delimiter::Brace => {
+            (Some(group.clone()), None)
+        }
+        TokenTree::Punct(punct) if punct.as_char() == ';' => (None, Some(punct.clone())),
+        _ => panic!("cannot parse function"),
+    };
+
+    Function {
+        attributes,
+        vis_marker,
+        qualifiers,
+        tk_fn_keyword,
+        name: fn_name,
+        generic_params,
+        tk_params_parens,
+        params,
+        where_clause,
+        tk_return_arrow,
+        return_ty,
+        tk_semicolon,
+        body: function_body,
+    }
+}
diff --git a/src/snapshots/venial__tests__parse_all_kw_fn.snap b/src/snapshots/venial__tests__parse_all_kw_fn.snap
@@ -40,6 +40,9 @@ Function(
                 },
             ),
         },
+        tk_fn_keyword: Ident(
+            fn,
+        ),
         name: Ident(
             all_kw,
         ),
@@ -66,6 +69,7 @@ Function(
         where_clause: None,
         tk_return_arrow: None,
         return_ty: None,
+        tk_semicolon: None,
         body: Some(
             Group {
                 delimiter: Brace,

diff --git a/src/snapshots/venial__tests__parse_async_fn.snap b/src/snapshots/venial__tests__parse_async_fn.snap
@@ -20,6 +20,9 @@ Function(
             tk_extern: None,
             extern_abi: None,
         },
+        tk_fn_keyword: Ident(
+            fn,
+        ),
         name: Ident(
             async_fn,
         ),
@@ -46,6 +49,7 @@ Function(
         where_clause: None,
         tk_return_arrow: None,
         return_ty: None,
+        tk_semicolon: None,
         body: Some(
             Group {
                 delimiter: Brace,

diff --git a/src/snapshots/venial__tests__parse_attr_fn.snap b/src/snapshots/venial__tests__parse_attr_fn.snap
@@ -26,6 +26,9 @@ Function(
             tk_extern: None,
             extern_abi: None,
         },
+        tk_fn_keyword: Ident(
+            fn,
+        ),
         name: Ident(
             my_attr_fn,
         ),
@@ -52,6 +55,7 @@ Function(
         where_clause: None,
         tk_return_arrow: None,
         return_ty: None,
+        tk_semicolon: None,
         body: Some(
             Group {
                 delimiter: Brace,

diff --git a/src/snapshots/venial__tests__parse_const_fn.snap b/src/snapshots/venial__tests__parse_const_fn.snap
@@ -20,6 +20,9 @@ Function(
             tk_extern: None,
             extern_abi: None,
         },
+        tk_fn_keyword: Ident(
+            fn,
+        ),
         name: Ident(
             const_fn,
         ),
@@ -46,6 +49,7 @@ Function(
         where_clause: None,
         tk_return_arrow: None,
         return_ty: None,
+        tk_semicolon: None,
         body: Some(
             Group {
                 delimiter: Brace,

diff --git a/src/snapshots/venial__tests__parse_default_fn.snap b/src/snapshots/venial__tests__parse_default_fn.snap
@@ -20,6 +20,9 @@ Function(
             tk_extern: None,
             extern_abi: None,
         },
+        tk_fn_keyword: Ident(
+            fn,
+        ),
         name: Ident(
             default_fn,
         ),
@@ -46,6 +49,7 @@ Function(
         where_clause: None,
         tk_return_arrow: None,
         return_ty: None,
+        tk_semicolon: None,
         body: Some(
             Group {
                 delimiter: Brace,

diff --git a/src/snapshots/venial__tests__parse_empty_fn.snap b/src/snapshots/venial__tests__parse_empty_fn.snap
@@ -14,6 +14,9 @@ Function(
             tk_extern: None,
             extern_abi: None,
         },
+        tk_fn_keyword: Ident(
+            fn,
+        ),
         name: Ident(
             test_me,
         ),
@@ -23,6 +26,7 @@ Function(
         where_clause: None,
         tk_return_arrow: None,
         return_ty: None,
+        tk_semicolon: None,
         body: Some(
             Group {
                 delimiter: Brace,

diff --git a/src/snapshots/venial__tests__parse_extern_abi_fn.snap b/src/snapshots/venial__tests__parse_extern_abi_fn.snap
@@ -24,6 +24,9 @@ Function(
                 },
             ),
         },
+        tk_fn_keyword: Ident(
+            fn,
+        ),
         name: Ident(
             extern_fn,
         ),
@@ -50,6 +53,7 @@ Function(
         where_clause: None,
         tk_return_arrow: None,
         return_ty: None,
+        tk_semicolon: None,
         body: Some(
             Group {
                 delimiter: Brace,

diff --git a/src/snapshots/venial__tests__parse_extern_fn.snap b/src/snapshots/venial__tests__parse_extern_fn.snap
@@ -20,6 +20,9 @@ Function(
             ),
             extern_abi: None,
         },
+        tk_fn_keyword: Ident(
+            fn,
+        ),
         name: Ident(
             extern_fn,
         ),
@@ -46,6 +49,7 @@ Function(
         where_clause: None,
         tk_return_arrow: None,
         return_ty: None,
+        tk_semicolon: None,
         body: Some(
             Group {
                 delimiter: Brace,

diff --git a/src/snapshots/venial__tests__parse_fn.snap b/src/snapshots/venial__tests__parse_fn.snap
@@ -14,6 +14,9 @@ Function(
             tk_extern: None,
             extern_abi: None,
         },
+        tk_fn_keyword: Ident(
+            fn,
+        ),
         name: Ident(
             hello,
         ),
@@ -71,6 +74,7 @@ Function(
                 String,
             ],
         ),
+        tk_semicolon: None,
         body: Some(
             Group {
                 delimiter: Brace,

diff --git a/src/snapshots/venial__tests__parse_fn_body.snap b/src/snapshots/venial__tests__parse_fn_body.snap
@@ -14,6 +14,9 @@ Function(
             tk_extern: None,
             extern_abi: None,
         },
+        tk_fn_keyword: Ident(
+            fn,
+        ),
         name: Ident(
             hello_world,
         ),
@@ -71,6 +74,7 @@ Function(
                 String,
             ],
         ),
+        tk_semicolon: None,
         body: Some(
             Group {
                 delimiter: Brace,