From 2a4a390f5aea9df44f503b3e8a1ce10558858f2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donny/=EA=B0=95=EB=8F=99=EC=9C=A4?= Date: Sat, 11 Mar 2023 19:12:38 +0900 Subject: [PATCH 01/24] Add table.rs --- crates/swc_ecma_parser/src/lexer/mod.rs | 1 + crates/swc_ecma_parser/src/lexer/table.rs | 26 +++++++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 crates/swc_ecma_parser/src/lexer/table.rs diff --git a/crates/swc_ecma_parser/src/lexer/mod.rs b/crates/swc_ecma_parser/src/lexer/mod.rs index 11efbc5d3132..04d6612c3211 100644 --- a/crates/swc_ecma_parser/src/lexer/mod.rs +++ b/crates/swc_ecma_parser/src/lexer/mod.rs @@ -25,6 +25,7 @@ pub mod input; mod jsx; mod number; mod state; +mod table; #[cfg(test)] mod tests; pub mod util; diff --git a/crates/swc_ecma_parser/src/lexer/table.rs b/crates/swc_ecma_parser/src/lexer/table.rs new file mode 100644 index 000000000000..0af4c4a7f61c --- /dev/null +++ b/crates/swc_ecma_parser/src/lexer/table.rs @@ -0,0 +1,26 @@ +use super::Lexer; + +type ByteHandler = Option fn(&mut Lexer<'aa>)>; + +/// Lookup table mapping any incoming byte to a handler function defined below. +static BYTE_HANDLERS: [ByteHandler; 256] = [ + // 0 1 2 3 4 5 6 7 8 9 A B C D E F // + EOF, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, // 0 + ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, // 1 + ___, EXL, QOT, ERR, IDT, PRC, AMP, QOT, PNO, PNC, ATR, PLS, COM, MIN, PRD, SLH, // 2 + ZER, DIG, DIG, DIG, DIG, DIG, DIG, DIG, DIG, DIG, COL, SEM, LSS, EQL, MOR, QST, // 3 + ERR, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, // 4 + IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, BTO, IDT, BTC, CRT, IDT, // 5 + TPL, IDT, L_B, L_C, L_D, L_E, L_F, IDT, IDT, L_I, IDT, IDT, L_L, IDT, L_N, IDT, // 6 + L_P, IDT, L_R, L_S, L_T, L_U, L_V, L_W, IDT, L_Y, IDT, BEO, PIP, BEC, TLD, ERR, // 7 + UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, // 8 + UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, // 9 + UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, // A + UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, // B + UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, // C + UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, // D + UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, // E + UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, // F +]; + +const ___: ByteHandler = None; From b6a67b5fc87016235ca9fb33630c31166d7ec894 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donny/=EA=B0=95=EB=8F=99=EC=9C=A4?= Date: Sat, 11 Mar 2023 19:20:47 +0900 Subject: [PATCH 02/24] More table --- crates/swc_ecma_parser/src/lexer/table.rs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/crates/swc_ecma_parser/src/lexer/table.rs b/crates/swc_ecma_parser/src/lexer/table.rs index 0af4c4a7f61c..869e78fbc6af 100644 --- a/crates/swc_ecma_parser/src/lexer/table.rs +++ b/crates/swc_ecma_parser/src/lexer/table.rs @@ -1,6 +1,13 @@ -use super::Lexer; +//! Lookup table for byte handlers. +//! +//! Idea is taken from ratel. +//! +//! https://github.com/ratel-rust/ratel-core/blob/e55a1310ba69a3f5ce2a9a6eef643feced02ac08/ratel/src/lexer/mod.rs#L665 -type ByteHandler = Option fn(&mut Lexer<'aa>)>; +use super::{LexResult, Lexer}; +use crate::token::Token; + +type ByteHandler = Option fn(&mut Lexer<'aa>) -> LexResult>>; /// Lookup table mapping any incoming byte to a handler function defined below. static BYTE_HANDLERS: [ByteHandler; 256] = [ @@ -24,3 +31,11 @@ static BYTE_HANDLERS: [ByteHandler; 256] = [ ]; const ___: ByteHandler = None; + +const EOF: ByteHandler = Some(|lexer| { + lexer.input.bump_bytes(1); + + Ok(None) +}); + +const IDT: ByteHandler = Some(|lexer| lexer.read_ident_or_keyword().map(Some)); From 528344e734f917b0cdb92ade486c43c6bdbb86f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donny/=EA=B0=95=EB=8F=99=EC=9C=A4?= Date: Sat, 11 Mar 2023 19:24:26 +0900 Subject: [PATCH 03/24] More --- crates/swc_ecma_parser/src/lexer/table.rs | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/crates/swc_ecma_parser/src/lexer/table.rs b/crates/swc_ecma_parser/src/lexer/table.rs index 869e78fbc6af..4dbdb4ef268a 100644 --- a/crates/swc_ecma_parser/src/lexer/table.rs +++ b/crates/swc_ecma_parser/src/lexer/table.rs @@ -4,6 +4,8 @@ //! //! https://github.com/ratel-rust/ratel-core/blob/e55a1310ba69a3f5ce2a9a6eef643feced02ac08/ratel/src/lexer/mod.rs#L665 +use either::Either; + use super::{LexResult, Lexer}; use crate::token::Token; @@ -15,7 +17,7 @@ static BYTE_HANDLERS: [ByteHandler; 256] = [ EOF, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, // 0 ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, // 1 ___, EXL, QOT, ERR, IDT, PRC, AMP, QOT, PNO, PNC, ATR, PLS, COM, MIN, PRD, SLH, // 2 - ZER, DIG, DIG, DIG, DIG, DIG, DIG, DIG, DIG, DIG, COL, SEM, LSS, EQL, MOR, QST, // 3 + ZER, DI0, DIG, DIG, DIG, DIG, DIG, DIG, DIG, DIG, COL, SEM, LSS, EQL, MOR, QST, // 3 ERR, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, // 4 IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, BTO, IDT, BTC, CRT, IDT, // 5 TPL, IDT, L_B, L_C, L_D, L_E, L_F, IDT, IDT, L_I, IDT, IDT, L_L, IDT, L_N, IDT, // 6 @@ -38,4 +40,22 @@ const EOF: ByteHandler = Some(|lexer| { Ok(None) }); +/// Identifier const IDT: ByteHandler = Some(|lexer| lexer.read_ident_or_keyword().map(Some)); + +/// `0` +const DI0: ByteHandler = Some(|lexer| lexer.read_token_zero().map(Some)); + +/// Numbers +const DIG: ByteHandler = Some(|lexer| { + lexer + .read_number(false) + .map(|v| match v { + Either::Left((value, raw)) => Token::Num { value, raw }, + Either::Right((value, raw)) => Token::BigInt { value, raw }, + }) + .map(Some) +}); + +/// String literals with `'` or `"` +const QOT: ByteHandler = Some(|lexer| self.read_str_lit().map(Some)); From 48b3bd78a5557c1d4eaba0786d78948516b4db18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donny/=EA=B0=95=EB=8F=99=EC=9C=A4?= Date: Sat, 11 Mar 2023 19:27:22 +0900 Subject: [PATCH 04/24] Unicode --- crates/swc_ecma_parser/src/lexer/table.rs | 25 ++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/crates/swc_ecma_parser/src/lexer/table.rs b/crates/swc_ecma_parser/src/lexer/table.rs index 4dbdb4ef268a..9af105240a79 100644 --- a/crates/swc_ecma_parser/src/lexer/table.rs +++ b/crates/swc_ecma_parser/src/lexer/table.rs @@ -5,9 +5,10 @@ //! https://github.com/ratel-rust/ratel-core/blob/e55a1310ba69a3f5ce2a9a6eef643feced02ac08/ratel/src/lexer/mod.rs#L665 use either::Either; +use swc_common::input::Input; -use super::{LexResult, Lexer}; -use crate::token::Token; +use super::{pos_span, util::CharExt, LexResult, Lexer}; +use crate::{error::SyntaxError, token::Token}; type ByteHandler = Option fn(&mut Lexer<'aa>) -> LexResult>>; @@ -58,4 +59,22 @@ const DIG: ByteHandler = Some(|lexer| { }); /// String literals with `'` or `"` -const QOT: ByteHandler = Some(|lexer| self.read_str_lit().map(Some)); +const QOT: ByteHandler = Some(|lexer| lexer.read_str_lit().map(Some)); + +/// Unicode +const UNI: ByteHandler = Some(|lexer| { + let c = unsafe { + // Safety: Byte handler is only called for non-last chracters + lexer.input.cur().unwrap_unchecked() + }; + + // Identifier or keyword. '\uXXXX' sequences are allowed in + // identifiers, so '\' also dispatches to that. + if c == '\\' || c.is_ident_start() { + return lexer.read_ident_or_keyword().map(Some); + } + + let start = lexer.cur_pos(); + lexer.input.bump(); + lexer.error_span(pos_span(start), SyntaxError::UnexpectedChar { c })? +}); From a1092c80a98ebc1a6aeda4b28e48839b30dc9444 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donny/=EA=B0=95=EB=8F=99=EC=9C=A4?= Date: Sat, 11 Mar 2023 19:28:14 +0900 Subject: [PATCH 05/24] Error --- crates/swc_ecma_parser/src/lexer/table.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/crates/swc_ecma_parser/src/lexer/table.rs b/crates/swc_ecma_parser/src/lexer/table.rs index 9af105240a79..b863939a176a 100644 --- a/crates/swc_ecma_parser/src/lexer/table.rs +++ b/crates/swc_ecma_parser/src/lexer/table.rs @@ -41,6 +41,17 @@ const EOF: ByteHandler = Some(|lexer| { Ok(None) }); +const ERR: ByteHandler = Some(|lexer| { + let c = unsafe { + // Safety: Byte handler is only called for non-last chracters + lexer.input.cur().unwrap_unchecked() + }; + + let start = lexer.cur_pos(); + lexer.input.bump(); + lexer.error_span(pos_span(start), SyntaxError::UnexpectedChar { c })? +}); + /// Identifier const IDT: ByteHandler = Some(|lexer| lexer.read_ident_or_keyword().map(Some)); From f871e8b88039e1e61e95defe21dc70e5c06b21cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donny/=EA=B0=95=EB=8F=99=EC=9C=A4?= Date: Sat, 11 Mar 2023 19:34:50 +0900 Subject: [PATCH 06/24] More --- crates/swc_ecma_parser/src/lexer/table.rs | 26 +++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/crates/swc_ecma_parser/src/lexer/table.rs b/crates/swc_ecma_parser/src/lexer/table.rs index b863939a176a..ca05ceb3d880 100644 --- a/crates/swc_ecma_parser/src/lexer/table.rs +++ b/crates/swc_ecma_parser/src/lexer/table.rs @@ -89,3 +89,29 @@ const UNI: ByteHandler = Some(|lexer| { lexer.input.bump(); lexer.error_span(pos_span(start), SyntaxError::UnexpectedChar { c })? }); + +/// `:` +const COL: ByteHandler = Some(|lexer| lexer.read_token_colon().map(Some)); + +macro_rules! single_char { + ($name:ident, $c:literal, $token:ident) => { + const $name: ByteHandler = Some(|lexer| { + lexer.input.bump_bytes(1); + Ok(Some(Token::$token)) + }); + }; +} + +single_char!(SEM, b';', Semi); +single_char!(COM, b',', Comma); + +single_char!(PNO, b'(', LParen); +single_char!(PNC, b')', RParen); + +single_char!(BTO, b'[', LBracket); +single_char!(BTC, b']', RBracket); + +single_char!(BEO, b'{', LBrace); +single_char!(BEC, b'}', RBrace); + +// b'@' | b'`' | b'~' From 337b634769bdb2493242870e1969c119d335e1ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donny/=EA=B0=95=EB=8F=99=EC=9C=A4?= Date: Sat, 11 Mar 2023 19:36:40 +0900 Subject: [PATCH 07/24] IDT --- crates/swc_ecma_parser/src/lexer/table.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/swc_ecma_parser/src/lexer/table.rs b/crates/swc_ecma_parser/src/lexer/table.rs index ca05ceb3d880..bc47c627244a 100644 --- a/crates/swc_ecma_parser/src/lexer/table.rs +++ b/crates/swc_ecma_parser/src/lexer/table.rs @@ -21,8 +21,8 @@ static BYTE_HANDLERS: [ByteHandler; 256] = [ ZER, DI0, DIG, DIG, DIG, DIG, DIG, DIG, DIG, DIG, COL, SEM, LSS, EQL, MOR, QST, // 3 ERR, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, // 4 IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, BTO, IDT, BTC, CRT, IDT, // 5 - TPL, IDT, L_B, L_C, L_D, L_E, L_F, IDT, IDT, L_I, IDT, IDT, L_L, IDT, L_N, IDT, // 6 - L_P, IDT, L_R, L_S, L_T, L_U, L_V, L_W, IDT, L_Y, IDT, BEO, PIP, BEC, TLD, ERR, // 7 + TPL, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, // 6 + IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, BEO, IDT, BEC, TLD, ERR, // 7 UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, // 8 UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, // 9 UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, // A From 3fe7b649ff1edd0c252da0a143dcf102b7c49464 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donny/=EA=B0=95=EB=8F=99=EC=9C=A4?= Date: Sat, 11 Mar 2023 19:37:44 +0900 Subject: [PATCH 08/24] More --- crates/swc_ecma_parser/src/lexer/table.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/swc_ecma_parser/src/lexer/table.rs b/crates/swc_ecma_parser/src/lexer/table.rs index bc47c627244a..2a4f1a5ded5f 100644 --- a/crates/swc_ecma_parser/src/lexer/table.rs +++ b/crates/swc_ecma_parser/src/lexer/table.rs @@ -104,6 +104,7 @@ macro_rules! single_char { single_char!(SEM, b';', Semi); single_char!(COM, b',', Comma); +single_char!(TPL, b'`', BackQuote); single_char!(PNO, b'(', LParen); single_char!(PNC, b')', RParen); @@ -113,5 +114,3 @@ single_char!(BTC, b']', RBracket); single_char!(BEO, b'{', LBrace); single_char!(BEC, b'}', RBrace); - -// b'@' | b'`' | b'~' From 2b01bbbb65ee460f6b84dfda18ce5aa4eae6ca2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donny/=EA=B0=95=EB=8F=99=EC=9C=A4?= Date: Sat, 11 Mar 2023 19:41:32 +0900 Subject: [PATCH 09/24] More --- crates/swc_ecma_parser/src/lexer/table.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/swc_ecma_parser/src/lexer/table.rs b/crates/swc_ecma_parser/src/lexer/table.rs index 2a4f1a5ded5f..864612bd9b35 100644 --- a/crates/swc_ecma_parser/src/lexer/table.rs +++ b/crates/swc_ecma_parser/src/lexer/table.rs @@ -93,6 +93,12 @@ const UNI: ByteHandler = Some(|lexer| { /// `:` const COL: ByteHandler = Some(|lexer| lexer.read_token_colon().map(Some)); +/// `%` +const PRC: ByteHandler = Some(|lexer| lexer.read_token_mul_mod(b'%').map(Some)); + +/// `*` +const ATR: ByteHandler = Some(|lexer| lexer.read_token_mul_mod(b'*').map(Some)); + macro_rules! single_char { ($name:ident, $c:literal, $token:ident) => { const $name: ByteHandler = Some(|lexer| { From 803220cc501a8b5f40cd1fe2bb0577da7642f6df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donny/=EA=B0=95=EB=8F=99=EC=9C=A4?= Date: Sat, 11 Mar 2023 19:42:20 +0900 Subject: [PATCH 10/24] More --- crates/swc_ecma_parser/src/lexer/table.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/swc_ecma_parser/src/lexer/table.rs b/crates/swc_ecma_parser/src/lexer/table.rs index 864612bd9b35..e071d571ae13 100644 --- a/crates/swc_ecma_parser/src/lexer/table.rs +++ b/crates/swc_ecma_parser/src/lexer/table.rs @@ -99,6 +99,9 @@ const PRC: ByteHandler = Some(|lexer| lexer.read_token_mul_mod(b'%').map(Some)); /// `*` const ATR: ByteHandler = Some(|lexer| lexer.read_token_mul_mod(b'*').map(Some)); +/// `?` +const QST: ByteHandler = Some(|lexer| lexer.read_token_question_mark().map(Some)); + macro_rules! single_char { ($name:ident, $c:literal, $token:ident) => { const $name: ByteHandler = Some(|lexer| { From 0a3345e5959e6717785d387e85fee89c3d7e23f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donny/=EA=B0=95=EB=8F=99=EC=9C=A4?= Date: Sat, 11 Mar 2023 19:43:44 +0900 Subject: [PATCH 11/24] More --- crates/swc_ecma_parser/src/lexer/table.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/swc_ecma_parser/src/lexer/table.rs b/crates/swc_ecma_parser/src/lexer/table.rs index e071d571ae13..950d0bba2983 100644 --- a/crates/swc_ecma_parser/src/lexer/table.rs +++ b/crates/swc_ecma_parser/src/lexer/table.rs @@ -102,6 +102,12 @@ const ATR: ByteHandler = Some(|lexer| lexer.read_token_mul_mod(b'*').map(Some)); /// `?` const QST: ByteHandler = Some(|lexer| lexer.read_token_question_mark().map(Some)); +/// `&` +const AMP: ByteHandler = Some(|lexer| lexer.read_token_logical(b'&').map(Some)); + +/// `|` +const PIP: ByteHandler = Some(|lexer| lexer.read_token_logical(b'|').map(Some)); + macro_rules! single_char { ($name:ident, $c:literal, $token:ident) => { const $name: ByteHandler = Some(|lexer| { From f3ae7b73e04c473f8848c127ab097b6368301f58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donny/=EA=B0=95=EB=8F=99=EC=9C=A4?= Date: Sat, 11 Mar 2023 19:46:56 +0900 Subject: [PATCH 12/24] ^ --- crates/swc_ecma_parser/src/lexer/table.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/crates/swc_ecma_parser/src/lexer/table.rs b/crates/swc_ecma_parser/src/lexer/table.rs index 950d0bba2983..5a08ae7c5794 100644 --- a/crates/swc_ecma_parser/src/lexer/table.rs +++ b/crates/swc_ecma_parser/src/lexer/table.rs @@ -8,7 +8,10 @@ use either::Either; use swc_common::input::Input; use super::{pos_span, util::CharExt, LexResult, Lexer}; -use crate::{error::SyntaxError, token::Token}; +use crate::{ + error::SyntaxError, + token::{AssignOpToken, BinOpToken, Token}, +}; type ByteHandler = Option fn(&mut Lexer<'aa>) -> LexResult>>; @@ -129,3 +132,15 @@ single_char!(BTC, b']', RBracket); single_char!(BEO, b'{', LBrace); single_char!(BEC, b'}', RBrace); + +/// `^` +const CRT: ByteHandler = Some(|lexer| { + // Bitwise xor + lexer.input.bump_bytes(1); + return Ok(Some(if lexer.input.cur_as_ascii() == Some(b'=') { + lexer.input.bump_bytes(1); + Token::AssignOp(AssignOpToken::BitXorAssign) + } else { + Token::BinOp(BinOpToken::BitXor) + })); +}); From a7071b417411b3e77df7b4d0b501f403e4322eda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donny/=EA=B0=95=EB=8F=99=EC=9C=A4?= Date: Sat, 11 Mar 2023 19:51:23 +0900 Subject: [PATCH 13/24] Refactor --- crates/swc_ecma_parser/src/lexer/mod.rs | 61 ++++++++++++------------- 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/crates/swc_ecma_parser/src/lexer/mod.rs b/crates/swc_ecma_parser/src/lexer/mod.rs index 04d6612c3211..597a27bdc16f 100644 --- a/crates/swc_ecma_parser/src/lexer/mod.rs +++ b/crates/swc_ecma_parser/src/lexer/mod.rs @@ -228,37 +228,7 @@ impl<'a> Lexer<'a> { })); } - b'+' | b'-' => { - let start = self.cur_pos(); - - self.input.bump(); - - // '++', '--' - return Ok(Some(if self.input.cur() == Some(c as char) { - self.input.bump(); - - // Handle --> - if self.state.had_line_break && c == b'-' && self.eat(b'>') { - self.emit_module_mode_error( - start, - SyntaxError::LegacyCommentInModule, - ); - self.skip_line_comment(0); - self.skip_space::()?; - return self.read_token(); - } - - if c == b'+' { - PlusPlus - } else { - MinusMinus - } - } else if self.input.eat_byte(b'=') { - AssignOp(if c == b'+' { AddAssign } else { SubAssign }) - } else { - BinOp(if c == b'+' { Add } else { Sub }) - })); - } + b'+' | b'-' => return self.read_token_plus_minus(c), b'<' | b'>' => return self.read_token_lt_gt(), @@ -696,6 +666,35 @@ impl<'a> Lexer<'a> { Ok(Some(vec![c.into()])) } + + fn read_token_plus_minus(&self, c: u8) -> LexResult> { + let start = self.cur_pos(); + + self.input.bump(); + + // '++', '--' + return Ok(Some(if self.input.cur() == Some(c as char) { + self.input.bump(); + + // Handle --> + if self.state.had_line_break && c == b'-' && self.eat(b'>') { + self.emit_module_mode_error(start, SyntaxError::LegacyCommentInModule); + self.skip_line_comment(0); + self.skip_space::()?; + return self.read_token(); + } + + if c == b'+' { + PlusPlus + } else { + MinusMinus + } + } else if self.input.eat_byte(b'=') { + AssignOp(if c == b'+' { AddAssign } else { SubAssign }) + } else { + BinOp(if c == b'+' { Add } else { Sub }) + })); + } } impl<'a> Lexer<'a> { From 164862208f339683949f743e7e29c2c0c058d69b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donny/=EA=B0=95=EB=8F=99=EC=9C=A4?= Date: Sat, 11 Mar 2023 19:52:22 +0900 Subject: [PATCH 14/24] Refactor: `read_token_bang_or_eq` --- crates/swc_ecma_parser/src/lexer/mod.rs | 83 ++++++++++++------------- 1 file changed, 41 insertions(+), 42 deletions(-) diff --git a/crates/swc_ecma_parser/src/lexer/mod.rs b/crates/swc_ecma_parser/src/lexer/mod.rs index 597a27bdc16f..f31977dcc438 100644 --- a/crates/swc_ecma_parser/src/lexer/mod.rs +++ b/crates/swc_ecma_parser/src/lexer/mod.rs @@ -232,48 +232,7 @@ impl<'a> Lexer<'a> { b'<' | b'>' => return self.read_token_lt_gt(), - b'!' | b'=' => { - let start = self.cur_pos(); - let had_line_break_before_last = self.had_line_break_before_last(); - - self.input.bump(); - - return Ok(Some(if self.input.eat_byte(b'=') { - // "==" - - if self.input.eat_byte(b'=') { - if c == b'!' { - BinOp(NotEqEq) - } else { - // ======= - // ^ - if had_line_break_before_last && self.is_str("====") { - self.emit_error_span( - fixed_len_span(start, 7), - SyntaxError::TS1185, - ); - self.skip_line_comment(4); - self.skip_space::()?; - return self.read_token(); - } - - BinOp(EqEqEq) - } - } else if c == b'!' { - BinOp(NotEq) - } else { - BinOp(EqEq) - } - } else if c == b'=' && self.input.eat_byte(b'>') { - // "=>" - - Arrow - } else if c == b'!' { - Bang - } else { - AssignOp(Assign) - })); - } + b'!' | b'=' => return self.read_token_bang_or_eq(c), b'a'..=b'z' | b'A'..=b'Z' | b'$' | b'_' | b'\\' => { // Fast path for ascii identifiers. @@ -695,6 +654,46 @@ impl<'a> Lexer<'a> { BinOp(if c == b'+' { Add } else { Sub }) })); } + + fn read_token_bang_or_eq(&self, c: u8) -> LexResult> { + let start = self.cur_pos(); + let had_line_break_before_last = self.had_line_break_before_last(); + + self.input.bump(); + + return Ok(Some(if self.input.eat_byte(b'=') { + // "==" + + if self.input.eat_byte(b'=') { + if c == b'!' { + BinOp(NotEqEq) + } else { + // ======= + // ^ + if had_line_break_before_last && self.is_str("====") { + self.emit_error_span(fixed_len_span(start, 7), SyntaxError::TS1185); + self.skip_line_comment(4); + self.skip_space::()?; + return self.read_token(); + } + + BinOp(EqEqEq) + } + } else if c == b'!' { + BinOp(NotEq) + } else { + BinOp(EqEq) + } + } else if c == b'=' && self.input.eat_byte(b'>') { + // "=>" + + Arrow + } else if c == b'!' { + Bang + } else { + AssignOp(Assign) + })); + } } impl<'a> Lexer<'a> { From a37073ed96c52a4860f202d1c0ebfa23dc5e4159 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donny/=EA=B0=95=EB=8F=99=EC=9C=A4?= Date: Sat, 11 Mar 2023 19:53:16 +0900 Subject: [PATCH 15/24] More table work --- crates/swc_ecma_parser/src/lexer/table.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/crates/swc_ecma_parser/src/lexer/table.rs b/crates/swc_ecma_parser/src/lexer/table.rs index 5a08ae7c5794..2e71f3e5d8aa 100644 --- a/crates/swc_ecma_parser/src/lexer/table.rs +++ b/crates/swc_ecma_parser/src/lexer/table.rs @@ -123,6 +123,7 @@ macro_rules! single_char { single_char!(SEM, b';', Semi); single_char!(COM, b',', Comma); single_char!(TPL, b'`', BackQuote); +single_char!(TLD, b'~', Tilde); single_char!(PNO, b'(', LParen); single_char!(PNC, b')', RParen); @@ -144,3 +145,15 @@ const CRT: ByteHandler = Some(|lexer| { Token::BinOp(BinOpToken::BitXor) })); }); + +/// `+` +const PLS: ByteHandler = Some(|lexer| lexer.read_token_plus_minus(b'+')); + +/// `-` +const MIN: ByteHandler = Some(|lexer| lexer.read_token_plus_minus(b'-')); + +/// `!` +const EXL: ByteHandler = Some(|lexer| lexer.read_token_bang_or_eq(b'!')); + +/// `=` +const EQL: ByteHandler = Some(|lexer| lexer.read_token_bang_or_eq(b'=')); From 9140d25b9bc348a155d0631c0a0fff20cf6d29a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donny/=EA=B0=95=EB=8F=99=EC=9C=A4?= Date: Sat, 11 Mar 2023 19:53:41 +0900 Subject: [PATCH 16/24] More table work --- crates/swc_ecma_parser/src/lexer/table.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/swc_ecma_parser/src/lexer/table.rs b/crates/swc_ecma_parser/src/lexer/table.rs index 2e71f3e5d8aa..60f3ed907bf3 100644 --- a/crates/swc_ecma_parser/src/lexer/table.rs +++ b/crates/swc_ecma_parser/src/lexer/table.rs @@ -21,7 +21,7 @@ static BYTE_HANDLERS: [ByteHandler; 256] = [ EOF, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, // 0 ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, // 1 ___, EXL, QOT, ERR, IDT, PRC, AMP, QOT, PNO, PNC, ATR, PLS, COM, MIN, PRD, SLH, // 2 - ZER, DI0, DIG, DIG, DIG, DIG, DIG, DIG, DIG, DIG, COL, SEM, LSS, EQL, MOR, QST, // 3 + ZER, DIG, DIG, DIG, DIG, DIG, DIG, DIG, DIG, DIG, COL, SEM, LSS, EQL, MOR, QST, // 3 ERR, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, // 4 IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, BTO, IDT, BTC, CRT, IDT, // 5 TPL, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, // 6 @@ -59,7 +59,7 @@ const ERR: ByteHandler = Some(|lexer| { const IDT: ByteHandler = Some(|lexer| lexer.read_ident_or_keyword().map(Some)); /// `0` -const DI0: ByteHandler = Some(|lexer| lexer.read_token_zero().map(Some)); +const ZER: ByteHandler = Some(|lexer| lexer.read_token_zero().map(Some)); /// Numbers const DIG: ByteHandler = Some(|lexer| { From 057d3f6caaa104f95a235a4d40d162a6a2d6d33b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donny/=EA=B0=95=EB=8F=99=EC=9C=A4?= Date: Sat, 11 Mar 2023 19:54:22 +0900 Subject: [PATCH 17/24] More table work --- crates/swc_ecma_parser/src/lexer/table.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/swc_ecma_parser/src/lexer/table.rs b/crates/swc_ecma_parser/src/lexer/table.rs index 60f3ed907bf3..e9a865d7b0e8 100644 --- a/crates/swc_ecma_parser/src/lexer/table.rs +++ b/crates/swc_ecma_parser/src/lexer/table.rs @@ -157,3 +157,6 @@ const EXL: ByteHandler = Some(|lexer| lexer.read_token_bang_or_eq(b'!')); /// `=` const EQL: ByteHandler = Some(|lexer| lexer.read_token_bang_or_eq(b'=')); + +/// `.` +const PRD: ByteHandler = Some(|lexer| lexer.read_token_dot().map(Some)); From 5babd45066303460c891e89ada617afca6335e08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donny/=EA=B0=95=EB=8F=99=EC=9C=A4?= Date: Sat, 11 Mar 2023 19:55:19 +0900 Subject: [PATCH 18/24] More table work --- crates/swc_ecma_parser/src/lexer/table.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/swc_ecma_parser/src/lexer/table.rs b/crates/swc_ecma_parser/src/lexer/table.rs index e9a865d7b0e8..ccaa292e798f 100644 --- a/crates/swc_ecma_parser/src/lexer/table.rs +++ b/crates/swc_ecma_parser/src/lexer/table.rs @@ -160,3 +160,9 @@ const EQL: ByteHandler = Some(|lexer| lexer.read_token_bang_or_eq(b'=')); /// `.` const PRD: ByteHandler = Some(|lexer| lexer.read_token_dot().map(Some)); + +/// `<` +const LSS: ByteHandler = Some(|lexer| lexer.read_token_lt_gt()); + +/// `>` +const MOR: ByteHandler = Some(|lexer| lexer.read_token_lt_gt()); From cd993468e15f7d93cd06bae697c78d73b875566c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donny/=EA=B0=95=EB=8F=99=EC=9C=A4?= Date: Sat, 11 Mar 2023 19:55:49 +0900 Subject: [PATCH 19/24] Table work is done --- crates/swc_ecma_parser/src/lexer/table.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/swc_ecma_parser/src/lexer/table.rs b/crates/swc_ecma_parser/src/lexer/table.rs index ccaa292e798f..858f1c804fc6 100644 --- a/crates/swc_ecma_parser/src/lexer/table.rs +++ b/crates/swc_ecma_parser/src/lexer/table.rs @@ -166,3 +166,6 @@ const LSS: ByteHandler = Some(|lexer| lexer.read_token_lt_gt()); /// `>` const MOR: ByteHandler = Some(|lexer| lexer.read_token_lt_gt()); + +/// `/` +const SLH: ByteHandler = Some(|lexer| lexer.read_slash()); From b0b30e68e70bbbfa85b7af4f5ac1e0f5a40fdcca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donny/=EA=B0=95=EB=8F=99=EC=9C=A4?= Date: Sat, 11 Mar 2023 19:56:26 +0900 Subject: [PATCH 20/24] Fix lexer --- crates/swc_ecma_parser/src/lexer/mod.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/swc_ecma_parser/src/lexer/mod.rs b/crates/swc_ecma_parser/src/lexer/mod.rs index f31977dcc438..4d45c3b99660 100644 --- a/crates/swc_ecma_parser/src/lexer/mod.rs +++ b/crates/swc_ecma_parser/src/lexer/mod.rs @@ -626,13 +626,13 @@ impl<'a> Lexer<'a> { Ok(Some(vec![c.into()])) } - fn read_token_plus_minus(&self, c: u8) -> LexResult> { + fn read_token_plus_minus(&mut self, c: u8) -> LexResult> { let start = self.cur_pos(); self.input.bump(); // '++', '--' - return Ok(Some(if self.input.cur() == Some(c as char) { + Ok(Some(if self.input.cur() == Some(c as char) { self.input.bump(); // Handle --> @@ -652,16 +652,16 @@ impl<'a> Lexer<'a> { AssignOp(if c == b'+' { AddAssign } else { SubAssign }) } else { BinOp(if c == b'+' { Add } else { Sub }) - })); + })) } - fn read_token_bang_or_eq(&self, c: u8) -> LexResult> { + fn read_token_bang_or_eq(&mut self, c: u8) -> LexResult> { let start = self.cur_pos(); let had_line_break_before_last = self.had_line_break_before_last(); self.input.bump(); - return Ok(Some(if self.input.eat_byte(b'=') { + Ok(Some(if self.input.eat_byte(b'=') { // "==" if self.input.eat_byte(b'=') { @@ -692,7 +692,7 @@ impl<'a> Lexer<'a> { Bang } else { AssignOp(Assign) - })); + })) } } From dca694405a1bf3a5534a409e579ebbeccdb0fa00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donny/=EA=B0=95=EB=8F=99=EC=9C=A4?= Date: Sat, 11 Mar 2023 20:02:44 +0900 Subject: [PATCH 21/24] Fix table --- crates/swc_ecma_parser/src/lexer/table.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/crates/swc_ecma_parser/src/lexer/table.rs b/crates/swc_ecma_parser/src/lexer/table.rs index 858f1c804fc6..4c1218e6ef65 100644 --- a/crates/swc_ecma_parser/src/lexer/table.rs +++ b/crates/swc_ecma_parser/src/lexer/table.rs @@ -13,19 +13,19 @@ use crate::{ token::{AssignOpToken, BinOpToken, Token}, }; -type ByteHandler = Option fn(&mut Lexer<'aa>) -> LexResult>>; +pub(super) type ByteHandler = Option fn(&mut Lexer<'aa>) -> LexResult>>; /// Lookup table mapping any incoming byte to a handler function defined below. -static BYTE_HANDLERS: [ByteHandler; 256] = [ +pub(super) static BYTE_HANDLERS: [ByteHandler; 256] = [ // 0 1 2 3 4 5 6 7 8 9 A B C D E F // EOF, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, // 0 ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, // 1 - ___, EXL, QOT, ERR, IDT, PRC, AMP, QOT, PNO, PNC, ATR, PLS, COM, MIN, PRD, SLH, // 2 + ___, EXL, QOT, HSH, IDT, PRC, AMP, QOT, PNO, PNC, ATR, PLS, COM, MIN, PRD, SLH, // 2 ZER, DIG, DIG, DIG, DIG, DIG, DIG, DIG, DIG, DIG, COL, SEM, LSS, EQL, MOR, QST, // 3 ERR, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, // 4 IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, BTO, IDT, BTC, CRT, IDT, // 5 TPL, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, // 6 - IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, BEO, IDT, BEC, TLD, ERR, // 7 + IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, BEO, PIP, BEC, TLD, ERR, // 7 UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, // 8 UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, // 9 UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, UNI, // A @@ -138,12 +138,12 @@ single_char!(BEC, b'}', RBrace); const CRT: ByteHandler = Some(|lexer| { // Bitwise xor lexer.input.bump_bytes(1); - return Ok(Some(if lexer.input.cur_as_ascii() == Some(b'=') { + Ok(Some(if lexer.input.cur_as_ascii() == Some(b'=') { lexer.input.bump_bytes(1); Token::AssignOp(AssignOpToken::BitXorAssign) } else { Token::BinOp(BinOpToken::BitXor) - })); + })) }); /// `+` @@ -169,3 +169,6 @@ const MOR: ByteHandler = Some(|lexer| lexer.read_token_lt_gt()); /// `/` const SLH: ByteHandler = Some(|lexer| lexer.read_slash()); + +/// `#` +const HSH: ByteHandler = Some(|lexer| lexer.read_token_number_sign()); From c1a7852f9a9e7669bcee5b977da0845b914f33b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donny/=EA=B0=95=EB=8F=99=EC=9C=A4?= Date: Sat, 11 Mar 2023 20:02:50 +0900 Subject: [PATCH 22/24] Use table --- crates/swc_ecma_parser/src/lexer/mod.rs | 113 ++++-------------------- 1 file changed, 17 insertions(+), 96 deletions(-) diff --git a/crates/swc_ecma_parser/src/lexer/mod.rs b/crates/swc_ecma_parser/src/lexer/mod.rs index 4d45c3b99660..0ea26c8fb3bf 100644 --- a/crates/swc_ecma_parser/src/lexer/mod.rs +++ b/crates/swc_ecma_parser/src/lexer/mod.rs @@ -9,7 +9,12 @@ use swc_atoms::{Atom, AtomGenerator}; use swc_common::{comments::Comments, input::StringInput, BytePos, Span}; use swc_ecma_ast::{op, EsVersion}; -use self::{comments_buffer::CommentsBuffer, state::State, util::*}; +use self::{ + comments_buffer::CommentsBuffer, + state::State, + table::{ByteHandler, BYTE_HANDLERS}, + util::*, +}; pub use self::{ input::Input, state::{TokenContext, TokenContexts}, @@ -162,107 +167,23 @@ impl<'a> Lexer<'a> { /// babel: `getTokenFromCode` fn read_token(&mut self) -> LexResult> { - let c = self.input.cur_as_ascii(); - - match c { - None => {} - Some(c) => { - match c { - b'#' => return self.read_token_number_sign(), - - // - b'.' => return self.read_token_dot().map(Some), - - b'(' | b')' | b';' | b',' | b'[' | b']' | b'{' | b'}' | b'@' | b'`' | b'~' => { - // These tokens are emitted directly. - self.input.bump(); - return Ok(Some(match c { - b'(' => LParen, - b')' => RParen, - b';' => Semi, - b',' => Comma, - b'[' => LBracket, - b']' => RBracket, - b'{' => LBrace, - b'}' => RBrace, - b'@' => At, - b'`' => tok!('`'), - b'~' => tok!('~'), - - _ => unreachable!(), - })); - } - - b'?' => return self.read_token_question_mark().map(Some), - - b':' => return self.read_token_colon().map(Some), - - b'0' => return self.read_token_zero().map(Some), - - b'1'..=b'9' => { - return self - .read_number(false) - .map(|v| match v { - Left((value, raw)) => Num { value, raw }, - Right((value, raw)) => BigInt { value, raw }, - }) - .map(Some); - } - - b'"' | b'\'' => return self.read_str_lit().map(Some), - - b'/' => return self.read_slash(), - - b'%' | b'*' => return self.read_token_mul_mod(c).map(Some), - - // Logical operators - b'|' | b'&' => return self.read_token_logical(c).map(Some), - b'^' => { - // Bitwise xor - self.input.bump(); - return Ok(Some(if self.input.cur() == Some('=') { - self.input.bump(); - AssignOp(BitXorAssign) - } else { - BinOp(BitXor) - })); - } - - b'+' | b'-' => return self.read_token_plus_minus(c), + let mut byte; - b'<' | b'>' => return self.read_token_lt_gt(), + loop { + byte = match self.input.as_str().as_bytes().first() { + Some(&v) => v, + None => return Ok(None), + }; - b'!' | b'=' => return self.read_token_bang_or_eq(c), + let handler = unsafe { *(&BYTE_HANDLERS as *const ByteHandler).offset(byte as isize) }; - b'a'..=b'z' | b'A'..=b'Z' | b'$' | b'_' | b'\\' => { - // Fast path for ascii identifiers. - return self.read_ident_or_keyword().map(Some); - } - _ => {} + match handler { + Some(handler) => return handler(self), + None => { + self.input.bump_bytes(1); } } } - - let c = match self.input.cur() { - Some(c) => c, - None => { - return Ok(None); - } - }; - - let token = { - // Identifier or keyword. '\uXXXX' sequences are allowed in - // identifiers, so '\' also dispatches to that. - if c == '\\' || c.is_ident_start() { - return self.read_ident_or_keyword().map(Some); - } - - let start = self.cur_pos(); - self.input.bump(); - self.error_span(pos_span(start), SyntaxError::UnexpectedChar { c })? - }; - - Ok(Some(token)) } /// `#` From 32794e83e81ea649b2b7f4442bcc51280dbea024 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donny/=EA=B0=95=EB=8F=99=EC=9C=A4?= Date: Sat, 11 Mar 2023 20:04:35 +0900 Subject: [PATCH 23/24] Fix table for `@` --- crates/swc_ecma_parser/src/lexer/table.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/swc_ecma_parser/src/lexer/table.rs b/crates/swc_ecma_parser/src/lexer/table.rs index 4c1218e6ef65..2e7072e6d523 100644 --- a/crates/swc_ecma_parser/src/lexer/table.rs +++ b/crates/swc_ecma_parser/src/lexer/table.rs @@ -22,7 +22,7 @@ pub(super) static BYTE_HANDLERS: [ByteHandler; 256] = [ ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, ___, // 1 ___, EXL, QOT, HSH, IDT, PRC, AMP, QOT, PNO, PNC, ATR, PLS, COM, MIN, PRD, SLH, // 2 ZER, DIG, DIG, DIG, DIG, DIG, DIG, DIG, DIG, DIG, COL, SEM, LSS, EQL, MOR, QST, // 3 - ERR, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, // 4 + AT_, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, // 4 IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, BTO, IDT, BTC, CRT, IDT, // 5 TPL, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, // 6 IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, BEO, PIP, BEC, TLD, ERR, // 7 @@ -124,6 +124,7 @@ single_char!(SEM, b';', Semi); single_char!(COM, b',', Comma); single_char!(TPL, b'`', BackQuote); single_char!(TLD, b'~', Tilde); +single_char!(AT_, b'@', At); single_char!(PNO, b'(', LParen); single_char!(PNC, b')', RParen); From a1f9a73f887099090da0b21ffa51dc8166160165 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donny/=EA=B0=95=EB=8F=99=EC=9C=A4?= Date: Sat, 11 Mar 2023 20:27:40 +0900 Subject: [PATCH 24/24] Fix test --- crates/swc_ecma_parser/src/lexer/mod.rs | 27 +++++++++++++------------ 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/crates/swc_ecma_parser/src/lexer/mod.rs b/crates/swc_ecma_parser/src/lexer/mod.rs index 0ea26c8fb3bf..05b855f18a2b 100644 --- a/crates/swc_ecma_parser/src/lexer/mod.rs +++ b/crates/swc_ecma_parser/src/lexer/mod.rs @@ -167,21 +167,22 @@ impl<'a> Lexer<'a> { /// babel: `getTokenFromCode` fn read_token(&mut self) -> LexResult> { - let mut byte; - - loop { - byte = match self.input.as_str().as_bytes().first() { - Some(&v) => v, - None => return Ok(None), - }; + let byte = match self.input.as_str().as_bytes().first() { + Some(&v) => v, + None => return Ok(None), + }; - let handler = unsafe { *(&BYTE_HANDLERS as *const ByteHandler).offset(byte as isize) }; + let handler = unsafe { *(&BYTE_HANDLERS as *const ByteHandler).offset(byte as isize) }; - match handler { - Some(handler) => return handler(self), - None => { - self.input.bump_bytes(1); - } + match handler { + Some(handler) => handler(self), + None => { + let start = self.cur_pos(); + self.input.bump_bytes(1); + self.error_span( + pos_span(start), + SyntaxError::UnexpectedChar { c: byte as _ }, + ) } } }