Fix parser (#727)

- Allow await in an yield expression (fixes #720) - Prevent duplicate tokens while capturing (fixes #726)
swc-project · Mar 25, 2020 · ebc7070 · ebc7070
1 parent b17b249
commit ebc7070
Show file tree

Hide file tree

Showing 9 changed files with 242 additions and 5 deletions.
diff --git a/ecmascript/parser/Cargo.toml b/ecmascript/parser/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "swc_ecma_parser"
-version = "0.21.6"
+version = "0.21.7"
 authors = ["강동윤 <kdy1997.dev@gmail.com>"]
 license = "Apache-2.0/MIT"
 repository = "https://github.com/swc-project/swc.git"

diff --git a/ecmascript/parser/src/lexer/state.rs b/ecmascript/parser/src/lexer/state.rs
@@ -116,6 +116,9 @@ impl<I: Input> Tokens for Lexer<'_, I> {
         self.target
     }
 
+    /// no-op, as `Lexer` does not use `Rc<RefCelll<T>>`.
+    fn revert(&mut self) {}
+
     fn set_expr_allowed(&mut self, allow: bool) {
         self.set_expr_allowed(allow)
     }

diff --git a/ecmascript/parser/src/parser/input.rs b/ecmascript/parser/src/parser/input.rs
@@ -14,6 +14,10 @@ pub trait Tokens: Clone + Iterator<Item = TokenAndSpan> {
     fn syntax(&self) -> Syntax;
     fn target(&self) -> JscTarget;
 
+    /// Revert to lastest clone. THis method exists to removed captured token
+    /// while backtracking.
+    fn revert(&mut self);
+
     fn set_expr_allowed(&mut self, allow: bool);
     fn token_context(&self) -> &lexer::TokenContexts;
     fn token_context_mut(&mut self) -> &mut lexer::TokenContexts;
@@ -65,6 +69,9 @@ impl Tokens for TokensInput {
         self.target
     }
 
+    /// no-op, as `TokensInput` does not use `Rc<RefCelll<T>>`.
+    fn revert(&mut self) {}
+
     fn set_expr_allowed(&mut self, _: bool) {}
 
     fn token_context(&self) -> &TokenContexts {
@@ -81,16 +88,28 @@ impl Tokens for TokensInput {
 }
 
 /// Note: Lexer need access to parser's context to lex correctly.
-#[derive(Debug, Clone)]
+#[derive(Debug)]
 pub struct Capturing<I: Tokens> {
     inner: I,
+    last_clone_idx: usize,
     captured: Rc<RefCell<Vec<TokenAndSpan>>>,
 }
 
+impl<I: Tokens> Clone for Capturing<I> {
+    fn clone(&self) -> Self {
+        Capturing {
+            last_clone_idx: self.captured.borrow().len(),
+            inner: self.inner.clone(),
+            captured: self.captured.clone(),
+        }
+    }
+}
+
 impl<I: Tokens> Capturing<I> {
     pub fn new(input: I) -> Self {
         Capturing {
             inner: input,
+            last_clone_idx: 0,
             captured: Default::default(),
         }
     }
@@ -127,6 +146,12 @@ impl<I: Tokens> Tokens for Capturing<I> {
         self.inner.target()
     }
 
+    fn revert(&mut self) {
+        self.inner.revert();
+        let len = self.last_clone_idx;
+        self.captured.borrow_mut().truncate(len);
+    }
+
     fn set_expr_allowed(&mut self, allow: bool) {
         self.inner.set_expr_allowed(allow)
     }
@@ -171,6 +196,10 @@ impl<I: Tokens> Buffer<I> {
         }
     }
 
+    pub fn revert(&mut self) {
+        self.iter.revert()
+    }
+
     pub fn store(&mut self, token: Token) {
         debug_assert!(self.next.is_none());
         debug_assert!(self.cur.is_none());

diff --git a/ecmascript/parser/src/parser/typescript.rs b/ecmascript/parser/src/parser/typescript.rs
@@ -473,6 +473,7 @@ impl<'a, I: Tokens> Parser<'a, I> {
         match res {
             Ok(Some(res)) if res => {
                 *self = cloned;
+                self.input.revert();
                 self.emit_err = true;
                 Ok(res)
             }
@@ -498,6 +499,7 @@ impl<'a, I: Tokens> Parser<'a, I> {
         match res {
             Ok(Some(res)) => {
                 *self = cloned;
+                self.input.revert();
                 self.emit_err = true;
                 Some(res)
             }
@@ -1014,7 +1016,9 @@ impl<'a, I: Tokens> Parser<'a, I> {
 
         let mut cloned = self.clone();
         cloned.emit_err = false;
-        op(&mut cloned)
+        let res = op(&mut cloned);
+        cloned.input.revert();
+        res
     }
 
     /// `tsIsUnambiguouslyStartOfFunctionType`
@@ -2320,7 +2324,10 @@ fn make_decl_declare(mut decl: Decl) -> Decl {
 
 #[cfg(test)]
 mod tests {
-    use crate::{test_parser, Syntax};
+    use crate::{
+        lexer::Lexer, test_parser, token::TokenAndSpan, Capturing, JscTarget, Parser, Syntax,
+        TsConfig,
+    };
     use swc_common::DUMMY_SP;
     use swc_ecma_ast::*;
     use testing::assert_eq_ignore_span;
@@ -2393,4 +2400,34 @@ mod tests {
 
         assert_eq_ignore_span!(actual, expected);
     }
+
+    #[test]
+    fn issue_726() {
+        crate::with_test_sess(
+            "type Test = (
+    string | number);",
+            |sess, input| {
+                let lexer = Lexer::new(
+                    sess,
+                    Syntax::Typescript(TsConfig {
+                        ..Default::default()
+                    }),
+                    JscTarget::Es2019,
+                    input,
+                    None,
+                );
+                let lexer = Capturing::new(lexer);
+
+                let mut parser = Parser::new_from(sess, lexer);
+                parser.parse_typescript_module().map_err(|mut e| {
+                    e.emit();
+                })?;
+                let tokens: Vec<TokenAndSpan> = parser.input().take();
+                let tokens = tokens.into_iter().map(|t| t.token).collect::<Vec<_>>();
+                assert_eq!(tokens.len(), 9, "Tokens: {:#?}", tokens);
+                Ok(())
+            },
+        )
+        .unwrap();
+    }
 }
diff --git a/ecmascript/parser/src/token.rs b/ecmascript/parser/src/token.rs
@@ -375,7 +375,7 @@ impl Debug for Word {
 #[kind(function(before_expr = "bool", starts_expr = "bool"))]
 pub enum Keyword {
     /// Spec says this might be identifier.
-    #[kind(before_expr)]
+    #[kind(before_expr, starts_expr)]
     Await,
 
     Break,

diff --git a/ecmascript/parser/tests/jsx/basic/custom/issue-720/input.js b/ecmascript/parser/tests/jsx/basic/custom/issue-720/input.js
@@ -0,0 +1,3 @@
+async function* main() {
+    yield await 0;
+}
diff --git a/ecmascript/parser/tests/jsx/basic/custom/issue-720/input.js.json b/ecmascript/parser/tests/jsx/basic/custom/issue-720/input.js.json
@@ -0,0 +1,81 @@
+{
+  "type": "Module",
+  "span": {
+    "start": 0,
+    "end": 45,
+    "ctxt": 0
+  },
+  "body": [
+    {
+      "type": "FunctionDeclaration",
+      "identifier": {
+        "type": "Identifier",
+        "span": {
+          "start": 16,
+          "end": 20,
+          "ctxt": 0
+        },
+        "value": "main",
+        "typeAnnotation": null,
+        "optional": false
+      },
+      "declare": false,
+      "params": [],
+      "decorators": [],
+      "span": {
+        "start": 0,
+        "end": 45,
+        "ctxt": 0
+      },
+      "body": {
+        "type": "BlockStatement",
+        "span": {
+          "start": 23,
+          "end": 45,
+          "ctxt": 0
+        },
+        "stmts": [
+          {
+            "type": "ExpressionStatement",
+            "span": {
+              "start": 29,
+              "end": 43,
+              "ctxt": 0
+            },
+            "expression": {
+              "type": "YieldExpression",
+              "span": {
+                "start": 29,
+                "end": 42,
+                "ctxt": 0
+              },
+              "argument": {
+                "type": "AwaitExpression",
+                "span": {
+                  "start": 35,
+                  "end": 42,
+                  "ctxt": 0
+                },
+                "argument": {
+                  "type": "NumericLiteral",
+                  "span": {
+                    "start": 41,
+                    "end": 42,
+                    "ctxt": 0
+                  },
+                  "value": 0.0
+                }
+              },
+              "delegate": false
+            }
+          }
+        ]
+      },
+      "generator": true,
+      "async": true,
+      "typeParameters": null,
+      "returnType": null
+    }
+  ],
+  "interpreter": null
+}
diff --git a/ecmascript/parser/tests/typescript/custom/issue-720/input.ts b/ecmascript/parser/tests/typescript/custom/issue-720/input.ts
@@ -0,0 +1,3 @@
+async function* main() {
+    yield await 0;
+}
diff --git a/ecmascript/parser/tests/typescript/custom/issue-720/input.ts.json b/ecmascript/parser/tests/typescript/custom/issue-720/input.ts.json
@@ -0,0 +1,81 @@
+{
+  "type": "Module",
+  "span": {
+    "start": 0,
+    "end": 45,
+    "ctxt": 0
+  },
+  "body": [
+    {
+      "type": "FunctionDeclaration",
+      "identifier": {
+        "type": "Identifier",
+        "span": {
+          "start": 16,
+          "end": 20,
+          "ctxt": 0
+        },
+        "value": "main",
+        "typeAnnotation": null,
+        "optional": false
+      },
+      "declare": false,
+      "params": [],
+      "decorators": [],
+      "span": {
+        "start": 0,
+        "end": 45,
+        "ctxt": 0
+      },
+      "body": {
+        "type": "BlockStatement",
+        "span": {
+          "start": 23,
+          "end": 45,
+          "ctxt": 0
+        },
+        "stmts": [
+          {
+            "type": "ExpressionStatement",
+            "span": {
+              "start": 29,
+              "end": 43,
+              "ctxt": 0
+            },
+            "expression": {
+              "type": "YieldExpression",
+              "span": {
+                "start": 29,
+                "end": 42,
+                "ctxt": 0
+              },
+              "argument": {
+                "type": "AwaitExpression",
+                "span": {
+                  "start": 35,
+                  "end": 42,
+                  "ctxt": 0
+                },
+                "argument": {
+                  "type": "NumericLiteral",
+                  "span": {
+                    "start": 41,
+                    "end": 42,
+                    "ctxt": 0
+                  },
+                  "value": 0.0
+                }
+              },
+              "delegate": false
+            }
+          }
+        ]
+      },
+      "generator": true,
+      "async": true,
+      "typeParameters": null,
+      "returnType": null
+    }
+  ],
+  "interpreter": null
+}