From ebc7070d1fa53ae41904bfcd7ad48d8bf344c1b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B0=95=EB=8F=99=EC=9C=A4?= Date: Wed, 25 Mar 2020 17:40:05 +0900 Subject: [PATCH] Fix parser (#727) - Allow await in an yield expression (fixes #720) - Prevent duplicate tokens while capturing (fixes #726) --- ecmascript/parser/Cargo.toml | 2 +- ecmascript/parser/src/lexer/state.rs | 3 + ecmascript/parser/src/parser/input.rs | 31 ++++++- ecmascript/parser/src/parser/typescript.rs | 41 +++++++++- ecmascript/parser/src/token.rs | 2 +- .../tests/jsx/basic/custom/issue-720/input.js | 3 + .../jsx/basic/custom/issue-720/input.js.json | 81 +++++++++++++++++++ .../typescript/custom/issue-720/input.ts | 3 + .../typescript/custom/issue-720/input.ts.json | 81 +++++++++++++++++++ 9 files changed, 242 insertions(+), 5 deletions(-) create mode 100644 ecmascript/parser/tests/jsx/basic/custom/issue-720/input.js create mode 100644 ecmascript/parser/tests/jsx/basic/custom/issue-720/input.js.json create mode 100644 ecmascript/parser/tests/typescript/custom/issue-720/input.ts create mode 100644 ecmascript/parser/tests/typescript/custom/issue-720/input.ts.json diff --git a/ecmascript/parser/Cargo.toml b/ecmascript/parser/Cargo.toml index 88156d70d963..636543ed7179 100644 --- a/ecmascript/parser/Cargo.toml +++ b/ecmascript/parser/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "swc_ecma_parser" -version = "0.21.6" +version = "0.21.7" authors = ["강동윤 "] license = "Apache-2.0/MIT" repository = "https://github.com/swc-project/swc.git" diff --git a/ecmascript/parser/src/lexer/state.rs b/ecmascript/parser/src/lexer/state.rs index be2e7af59dab..40410397da11 100644 --- a/ecmascript/parser/src/lexer/state.rs +++ b/ecmascript/parser/src/lexer/state.rs @@ -116,6 +116,9 @@ impl Tokens for Lexer<'_, I> { self.target } + /// no-op, as `Lexer` does not use `Rc>`. + fn revert(&mut self) {} + fn set_expr_allowed(&mut self, allow: bool) { self.set_expr_allowed(allow) } diff --git a/ecmascript/parser/src/parser/input.rs b/ecmascript/parser/src/parser/input.rs index 54b2848c39f0..a30b7a316d6d 100644 --- a/ecmascript/parser/src/parser/input.rs +++ b/ecmascript/parser/src/parser/input.rs @@ -14,6 +14,10 @@ pub trait Tokens: Clone + Iterator { fn syntax(&self) -> Syntax; fn target(&self) -> JscTarget; + /// Revert to lastest clone. THis method exists to removed captured token + /// while backtracking. + fn revert(&mut self); + fn set_expr_allowed(&mut self, allow: bool); fn token_context(&self) -> &lexer::TokenContexts; fn token_context_mut(&mut self) -> &mut lexer::TokenContexts; @@ -65,6 +69,9 @@ impl Tokens for TokensInput { self.target } + /// no-op, as `TokensInput` does not use `Rc>`. + fn revert(&mut self) {} + fn set_expr_allowed(&mut self, _: bool) {} fn token_context(&self) -> &TokenContexts { @@ -81,16 +88,28 @@ impl Tokens for TokensInput { } /// Note: Lexer need access to parser's context to lex correctly. -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct Capturing { inner: I, + last_clone_idx: usize, captured: Rc>>, } +impl Clone for Capturing { + fn clone(&self) -> Self { + Capturing { + last_clone_idx: self.captured.borrow().len(), + inner: self.inner.clone(), + captured: self.captured.clone(), + } + } +} + impl Capturing { pub fn new(input: I) -> Self { Capturing { inner: input, + last_clone_idx: 0, captured: Default::default(), } } @@ -127,6 +146,12 @@ impl Tokens for Capturing { self.inner.target() } + fn revert(&mut self) { + self.inner.revert(); + let len = self.last_clone_idx; + self.captured.borrow_mut().truncate(len); + } + fn set_expr_allowed(&mut self, allow: bool) { self.inner.set_expr_allowed(allow) } @@ -171,6 +196,10 @@ impl Buffer { } } + pub fn revert(&mut self) { + self.iter.revert() + } + pub fn store(&mut self, token: Token) { debug_assert!(self.next.is_none()); debug_assert!(self.cur.is_none()); diff --git a/ecmascript/parser/src/parser/typescript.rs b/ecmascript/parser/src/parser/typescript.rs index 22c1cc7f0949..fe2493013925 100644 --- a/ecmascript/parser/src/parser/typescript.rs +++ b/ecmascript/parser/src/parser/typescript.rs @@ -473,6 +473,7 @@ impl<'a, I: Tokens> Parser<'a, I> { match res { Ok(Some(res)) if res => { *self = cloned; + self.input.revert(); self.emit_err = true; Ok(res) } @@ -498,6 +499,7 @@ impl<'a, I: Tokens> Parser<'a, I> { match res { Ok(Some(res)) => { *self = cloned; + self.input.revert(); self.emit_err = true; Some(res) } @@ -1014,7 +1016,9 @@ impl<'a, I: Tokens> Parser<'a, I> { let mut cloned = self.clone(); cloned.emit_err = false; - op(&mut cloned) + let res = op(&mut cloned); + cloned.input.revert(); + res } /// `tsIsUnambiguouslyStartOfFunctionType` @@ -2320,7 +2324,10 @@ fn make_decl_declare(mut decl: Decl) -> Decl { #[cfg(test)] mod tests { - use crate::{test_parser, Syntax}; + use crate::{ + lexer::Lexer, test_parser, token::TokenAndSpan, Capturing, JscTarget, Parser, Syntax, + TsConfig, + }; use swc_common::DUMMY_SP; use swc_ecma_ast::*; use testing::assert_eq_ignore_span; @@ -2393,4 +2400,34 @@ mod tests { assert_eq_ignore_span!(actual, expected); } + + #[test] + fn issue_726() { + crate::with_test_sess( + "type Test = ( + string | number);", + |sess, input| { + let lexer = Lexer::new( + sess, + Syntax::Typescript(TsConfig { + ..Default::default() + }), + JscTarget::Es2019, + input, + None, + ); + let lexer = Capturing::new(lexer); + + let mut parser = Parser::new_from(sess, lexer); + parser.parse_typescript_module().map_err(|mut e| { + e.emit(); + })?; + let tokens: Vec = parser.input().take(); + let tokens = tokens.into_iter().map(|t| t.token).collect::>(); + assert_eq!(tokens.len(), 9, "Tokens: {:#?}", tokens); + Ok(()) + }, + ) + .unwrap(); + } } diff --git a/ecmascript/parser/src/token.rs b/ecmascript/parser/src/token.rs index 40da89a06dcd..d80eeb43d6a0 100644 --- a/ecmascript/parser/src/token.rs +++ b/ecmascript/parser/src/token.rs @@ -375,7 +375,7 @@ impl Debug for Word { #[kind(function(before_expr = "bool", starts_expr = "bool"))] pub enum Keyword { /// Spec says this might be identifier. - #[kind(before_expr)] + #[kind(before_expr, starts_expr)] Await, Break, diff --git a/ecmascript/parser/tests/jsx/basic/custom/issue-720/input.js b/ecmascript/parser/tests/jsx/basic/custom/issue-720/input.js new file mode 100644 index 000000000000..31b85f4169b6 --- /dev/null +++ b/ecmascript/parser/tests/jsx/basic/custom/issue-720/input.js @@ -0,0 +1,3 @@ +async function* main() { + yield await 0; +} \ No newline at end of file diff --git a/ecmascript/parser/tests/jsx/basic/custom/issue-720/input.js.json b/ecmascript/parser/tests/jsx/basic/custom/issue-720/input.js.json new file mode 100644 index 000000000000..3db42939902d --- /dev/null +++ b/ecmascript/parser/tests/jsx/basic/custom/issue-720/input.js.json @@ -0,0 +1,81 @@ +{ + "type": "Module", + "span": { + "start": 0, + "end": 45, + "ctxt": 0 + }, + "body": [ + { + "type": "FunctionDeclaration", + "identifier": { + "type": "Identifier", + "span": { + "start": 16, + "end": 20, + "ctxt": 0 + }, + "value": "main", + "typeAnnotation": null, + "optional": false + }, + "declare": false, + "params": [], + "decorators": [], + "span": { + "start": 0, + "end": 45, + "ctxt": 0 + }, + "body": { + "type": "BlockStatement", + "span": { + "start": 23, + "end": 45, + "ctxt": 0 + }, + "stmts": [ + { + "type": "ExpressionStatement", + "span": { + "start": 29, + "end": 43, + "ctxt": 0 + }, + "expression": { + "type": "YieldExpression", + "span": { + "start": 29, + "end": 42, + "ctxt": 0 + }, + "argument": { + "type": "AwaitExpression", + "span": { + "start": 35, + "end": 42, + "ctxt": 0 + }, + "argument": { + "type": "NumericLiteral", + "span": { + "start": 41, + "end": 42, + "ctxt": 0 + }, + "value": 0.0 + } + }, + "delegate": false + } + } + ] + }, + "generator": true, + "async": true, + "typeParameters": null, + "returnType": null + } + ], + "interpreter": null +} diff --git a/ecmascript/parser/tests/typescript/custom/issue-720/input.ts b/ecmascript/parser/tests/typescript/custom/issue-720/input.ts new file mode 100644 index 000000000000..31b85f4169b6 --- /dev/null +++ b/ecmascript/parser/tests/typescript/custom/issue-720/input.ts @@ -0,0 +1,3 @@ +async function* main() { + yield await 0; +} \ No newline at end of file diff --git a/ecmascript/parser/tests/typescript/custom/issue-720/input.ts.json b/ecmascript/parser/tests/typescript/custom/issue-720/input.ts.json new file mode 100644 index 000000000000..3db42939902d --- /dev/null +++ b/ecmascript/parser/tests/typescript/custom/issue-720/input.ts.json @@ -0,0 +1,81 @@ +{ + "type": "Module", + "span": { + "start": 0, + "end": 45, + "ctxt": 0 + }, + "body": [ + { + "type": "FunctionDeclaration", + "identifier": { + "type": "Identifier", + "span": { + "start": 16, + "end": 20, + "ctxt": 0 + }, + "value": "main", + "typeAnnotation": null, + "optional": false + }, + "declare": false, + "params": [], + "decorators": [], + "span": { + "start": 0, + "end": 45, + "ctxt": 0 + }, + "body": { + "type": "BlockStatement", + "span": { + "start": 23, + "end": 45, + "ctxt": 0 + }, + "stmts": [ + { + "type": "ExpressionStatement", + "span": { + "start": 29, + "end": 43, + "ctxt": 0 + }, + "expression": { + "type": "YieldExpression", + "span": { + "start": 29, + "end": 42, + "ctxt": 0 + }, + "argument": { + "type": "AwaitExpression", + "span": { + "start": 35, + "end": 42, + "ctxt": 0 + }, + "argument": { + "type": "NumericLiteral", + "span": { + "start": 41, + "end": 42, + "ctxt": 0 + }, + "value": 0.0 + } + }, + "delegate": false + } + } + ] + }, + "generator": true, + "async": true, + "typeParameters": null, + "returnType": null + } + ], + "interpreter": null +}