Feat: add shebang handling to lexer and parser (#43)

* Feat: add read_shebang to lexer * Test: add shebang test * Feat: add shebang to AST node * Feat: eat shebang at the beginning of script/module * Fix: fix lookup table * Test: fix shebang test * Chore: update Cargo.lock * Docs: update CHANGELOG of lexer and parser * Fix: fix read_shebang in lexer * Test: add shebang test for `\u{2028}` * Chore: remove strip_shebang * Refactor: move shebang tese to be organized * Fix: fix span where error will be reported * Docs: add change log about removing `Lexer::strip_shebang` * Test: add more tests for shebang
rslint · Oct 13, 2020 · 4d2eaf1 · 4d2eaf1
1 parent 17bcc31
commit 4d2eaf1
Show file tree

Hide file tree

Showing 10 changed files with 139 additions and 32 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/crates/rslint_lexer/CHANGELOG.md b/crates/rslint_lexer/CHANGELOG.md
@@ -6,6 +6,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased]
 
+### Removed
+
+- Removed `Lexer::strip_shebang` in favor of proper shebang handling
+
+### Added
+
+- Added handling of shebang
+
 ### Fixed
 
 - Fixed handling of `/=` and `>>=`

diff --git a/crates/rslint_lexer/src/lib.rs b/crates/rslint_lexer/src/lib.rs
@@ -109,29 +109,6 @@ impl<'src> Lexer<'src> {
         }
     }
 
-    /// Strip away the possible shebang sequence of a source
-    /// **This is not automatically done by the lexer**
-    pub fn strip_shebang(&mut self) {
-        if let Some(b"#!") = self.bytes.get(0..2) {
-            // Safety: Calling strip_shebang in the middle of lexing can potentially cause undefined behavior
-            // because the cursor is a byte index, advancing blindly into a utf8 boundary is a big oopsie and
-            // can lead to undefined behavior, therefore we must return if the lexer is not at the start
-            if self.cur != 0 {
-                return;
-            }
-
-            self.next();
-            while self.next().is_some() {
-                let chr = self.get_unicode_char();
-                self.cur += chr.len_utf8() - 1;
-
-                if is_linebreak(chr) {
-                    return;
-                }
-            }
-        }
-    }
-
     // Bump the lexer and return the token given in
     fn eat(&mut self, tok: LexerReturn) -> LexerReturn {
         self.next();
@@ -757,6 +734,37 @@ impl<'src> Lexer<'src> {
         }
     }
 
+    #[inline]
+    fn read_shebang(&mut self) -> LexerReturn {
+        let start = self.cur;
+        self.next();
+        if start != 0 {
+            let err = Diagnostic::error()
+                .with_message("`#` must be at the beginning of the file")
+                .with_labels(vec![Label::primary(self.file_id, start..(start + 1))
+                    .with_message("but it's found here")]);
+            return (Token::new(SyntaxKind::ERROR_TOKEN, 1), Some(err));
+        }
+
+        if let Some(b'!') = self.bytes.get(1) {
+            while self.next().is_some() {
+                let chr = self.get_unicode_char();
+
+                if is_linebreak(chr) {
+                    return tok!(SHEBANG, self.cur);
+                }
+                self.cur += chr.len_utf8() - 1;
+            }
+            tok!(SHEBANG, self.cur)
+        } else {
+            let err = Diagnostic::error()
+                .with_message("expected `!` following a `#`, but found none")
+                .with_labels(vec![Label::primary(self.file_id, 0..1).with_message("")]);
+
+            (Token::new(SyntaxKind::ERROR_TOKEN, 1), Some(err))
+        }
+    }
+
     #[inline]
     fn read_slash(&mut self) -> LexerReturn {
         let start = self.cur;
@@ -1130,6 +1138,7 @@ impl<'src> Lexer<'src> {
                 tok!(WHITESPACE, self.cur - start)
             }
             EXL => self.resolve_bang(),
+            HAS => self.read_shebang(),
             PRC => self.bin_or_assign(T![%], T![%=]),
             AMP => self.resolve_amp(),
             PNO => self.eat(tok!(L_PAREN, 1)),
@@ -1366,6 +1375,7 @@ enum Dispatch {
     EXL,
     QOT,
     IDT,
+    HAS,
     PRC,
     AMP,
     PNO,
@@ -1418,7 +1428,7 @@ static DISPATCHER: [Dispatch; 256] = [
     //   0    1    2    3    4    5    6    7    8    9    A    B    C    D    E    F   //
     ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, WHS, WHS, WHS, WHS, WHS, ERR, ERR, // 0
     ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, // 1
-    WHS, EXL, QOT, ERR, IDT, PRC, AMP, QOT, PNO, PNC, MUL, PLS, COM, MIN, PRD, SLH, // 2
+    WHS, EXL, QOT, HAS, IDT, PRC, AMP, QOT, PNO, PNC, MUL, PLS, COM, MIN, PRD, SLH, // 2
     ZER, DIG, DIG, DIG, DIG, DIG, DIG, DIG, DIG, DIG, COL, SEM, LSS, EQL, MOR, QST, // 3
     ERR, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, // 4
     IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, BTO, BSL, BTC, CRT, IDT, // 5

diff --git a/crates/rslint_lexer/src/tests.rs b/crates/rslint_lexer/src/tests.rs
@@ -82,13 +82,6 @@ fn losslessness(string: String) -> bool {
     string == new_str
 }
 
-#[test]
-fn strip_shebang() {
-    let mut lex = Lexer::from_str("#! /bin/node \n\n", 0);
-    lex.strip_shebang();
-    assert_eq!(lex.cur, 13);
-}
-
 #[test]
 fn empty() {
     assert_lex! {
@@ -806,6 +799,40 @@ fn bigint_literals() {
     }
 }
 
+#[test]
+fn shebang() {
+    assert_lex! {
+        "#! /bin/node",
+        SHEBANG:12
+    }
+
+    assert_lex! {
+        "#!/bin/node\n",
+        SHEBANG:11,
+        WHITESPACE:1
+    }
+
+    assert_lex! {
+        "#!/usr/bin/env deno\u{2028}",
+        SHEBANG:19,
+        WHITESPACE:3
+    }
+
+    assert_lex! {
+        "#0",
+        ERROR_TOKEN:1,
+        NUMBER:1
+    }
+
+    assert_lex! {
+        "0#!/bin/deno",
+        NUMBER:1,
+        ERROR_TOKEN:1,
+        BANG:1,
+        REGEX:9
+    }
+}
+
 #[test]
 fn single_line_comments() {
     assert_lex! {

diff --git a/crates/rslint_parser/CHANGELOG.MD b/crates/rslint_parser/CHANGELOG.MD
@@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased]
 
+### Added
+
+- Added handling of shebang where the parser consumes shebang if exists at the start
+
 ### Fixed
 
 - Fixed handling of `/=` and `>>=`

diff --git a/crates/rslint_parser/src/ast/generated/nodes.rs b/crates/rslint_parser/src/ast/generated/nodes.rs
@@ -10,13 +10,15 @@ pub struct Script {
     pub(crate) syntax: SyntaxNode,
 }
 impl Script {
+    pub fn shebang_token(&self) -> Option<SyntaxToken> { support::token(&self.syntax, T![shebang]) }
     pub fn items(&self) -> AstChildren<Stmt> { support::children(&self.syntax) }
 }
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct Module {
     pub(crate) syntax: SyntaxNode,
 }
 impl Module {
+    pub fn shebang_token(&self) -> Option<SyntaxToken> { support::token(&self.syntax, T![shebang]) }
     pub fn items(&self) -> AstChildren<ModuleItem> { support::children(&self.syntax) }
 }
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]

diff --git a/crates/rslint_parser/src/syntax/program.rs b/crates/rslint_parser/src/syntax/program.rs
@@ -16,6 +16,7 @@ pub fn script(p: &mut Parser) -> CompletedMarker {
         "Using the script parsing function for modules is erroneous"
     );
     let m = p.start();
+    p.eat(T![shebang]);
     block_items(p, true, true, None);
     m.complete(p, SyntaxKind::SCRIPT)
 }
@@ -30,6 +31,7 @@ pub fn module(p: &mut Parser) -> CompletedMarker {
         "Using the module parsing function for scripts is erroneous"
     );
     let m = p.start();
+    p.eat(T![shebang]);
     block_items(p, true, true, None);
     m.complete(p, SyntaxKind::MODULE)
 }

diff --git a/crates/rslint_syntax/src/generated.rs b/crates/rslint_syntax/src/generated.rs
@@ -401,4 +401,4 @@ impl SyntaxKind {
 }
 #[doc = r" Utility macro for creating a SyntaxKind through simple macro syntax"]
 #[macro_export]
-macro_rules ! T { [ ; ] => { $ crate :: SyntaxKind :: SEMICOLON } ; [ , ] => { $ crate :: SyntaxKind :: COMMA } ; [ '(' ] => { $ crate :: SyntaxKind :: L_PAREN } ; [ ')' ] => { $ crate :: SyntaxKind :: R_PAREN } ; [ '{' ] => { $ crate :: SyntaxKind :: L_CURLY } ; [ '}' ] => { $ crate :: SyntaxKind :: R_CURLY } ; [ '[' ] => { $ crate :: SyntaxKind :: L_BRACK } ; [ ']' ] => { $ crate :: SyntaxKind :: R_BRACK } ; [ < ] => { $ crate :: SyntaxKind :: L_ANGLE } ; [ > ] => { $ crate :: SyntaxKind :: R_ANGLE } ; [ ~ ] => { $ crate :: SyntaxKind :: TILDE } ; [ ? ] => { $ crate :: SyntaxKind :: QUESTION } ; [ ?? ] => { $ crate :: SyntaxKind :: QUESTION2 } ; [ ?. ] => { $ crate :: SyntaxKind :: QUESTIONDOT } ; [ & ] => { $ crate :: SyntaxKind :: AMP } ; [ | ] => { $ crate :: SyntaxKind :: PIPE } ; [ + ] => { $ crate :: SyntaxKind :: PLUS } ; [ ++ ] => { $ crate :: SyntaxKind :: PLUS2 } ; [ * ] => { $ crate :: SyntaxKind :: STAR } ; [ ** ] => { $ crate :: SyntaxKind :: STAR2 } ; [ / ] => { $ crate :: SyntaxKind :: SLASH } ; [ ^ ] => { $ crate :: SyntaxKind :: CARET } ; [ % ] => { $ crate :: SyntaxKind :: PERCENT } ; [ . ] => { $ crate :: SyntaxKind :: DOT } ; [ ... ] => { $ crate :: SyntaxKind :: DOT2 } ; [ : ] => { $ crate :: SyntaxKind :: COLON } ; [ = ] => { $ crate :: SyntaxKind :: EQ } ; [ == ] => { $ crate :: SyntaxKind :: EQ2 } ; [ === ] => { $ crate :: SyntaxKind :: EQ3 } ; [ => ] => { $ crate :: SyntaxKind :: FAT_ARROW } ; [ ! ] => { $ crate :: SyntaxKind :: BANG } ; [ != ] => { $ crate :: SyntaxKind :: NEQ } ; [ !== ] => { $ crate :: SyntaxKind :: NEQ2 } ; [ - ] => { $ crate :: SyntaxKind :: MINUS } ; [ -- ] => { $ crate :: SyntaxKind :: MINUS2 } ; [ <= ] => { $ crate :: SyntaxKind :: LTEQ } ; [ >= ] => { $ crate :: SyntaxKind :: GTEQ } ; [ += ] => { $ crate :: SyntaxKind :: PLUSEQ } ; [ -= ] => { $ crate :: SyntaxKind :: MINUSEQ } ; [ |= ] => { $ crate :: SyntaxKind :: PIPEEQ } ; [ &= ] => { $ crate :: SyntaxKind :: AMPEQ } ; [ ^= ] => { $ crate :: SyntaxKind :: CARETEQ } ; [ /= ] => { $ crate :: SyntaxKind :: SLASHEQ } ; [ *= ] => { $ crate :: SyntaxKind :: STAREQ } ; [ %= ] => { $ crate :: SyntaxKind :: PERCENTEQ } ; [ && ] => { $ crate :: SyntaxKind :: AMP2 } ; [ || ] => { $ crate :: SyntaxKind :: PIPE2 } ; [ << ] => { $ crate :: SyntaxKind :: SHL } ; [ >> ] => { $ crate :: SyntaxKind :: SHR } ; [ >>> ] => { $ crate :: SyntaxKind :: USHR } ; [ <<= ] => { $ crate :: SyntaxKind :: SHLEQ } ; [ >>= ] => { $ crate :: SyntaxKind :: SHREQ } ; [ >>>= ] => { $ crate :: SyntaxKind :: USHREQ } ; [ &&= ] => { $ crate :: SyntaxKind :: AMP2EQ } ; [ ||= ] => { $ crate :: SyntaxKind :: PIPE2EQ } ; [ **= ] => { $ crate :: SyntaxKind :: STAR2EQ } ; [ ??= ] => { $ crate :: SyntaxKind :: QUESTION2EQ } ; [ await ] => { $ crate :: SyntaxKind :: AWAIT_KW } ; [ break ] => { $ crate :: SyntaxKind :: BREAK_KW } ; [ case ] => { $ crate :: SyntaxKind :: CASE_KW } ; [ catch ] => { $ crate :: SyntaxKind :: CATCH_KW } ; [ class ] => { $ crate :: SyntaxKind :: CLASS_KW } ; [ const ] => { $ crate :: SyntaxKind :: CONST_KW } ; [ continue ] => { $ crate :: SyntaxKind :: CONTINUE_KW } ; [ debugger ] => { $ crate :: SyntaxKind :: DEBUGGER_KW } ; [ default ] => { $ crate :: SyntaxKind :: DEFAULT_KW } ; [ delete ] => { $ crate :: SyntaxKind :: DELETE_KW } ; [ do ] => { $ crate :: SyntaxKind :: DO_KW } ; [ else ] => { $ crate :: SyntaxKind :: ELSE_KW } ; [ enum ] => { $ crate :: SyntaxKind :: ENUM_KW } ; [ export ] => { $ crate :: SyntaxKind :: EXPORT_KW } ; [ extends ] => { $ crate :: SyntaxKind :: EXTENDS_KW } ; [ false ] => { $ crate :: SyntaxKind :: FALSE_KW } ; [ finally ] => { $ crate :: SyntaxKind :: FINALLY_KW } ; [ for ] => { $ crate :: SyntaxKind :: FOR_KW } ; [ function ] => { $ crate :: SyntaxKind :: FUNCTION_KW } ; [ if ] => { $ crate :: SyntaxKind :: IF_KW } ; [ in ] => { $ crate :: SyntaxKind :: IN_KW } ; [ instanceof ] => { $ crate :: SyntaxKind :: INSTANCEOF_KW } ; [ interface ] => { $ crate :: SyntaxKind :: INTERFACE_KW } ; [ import ] => { $ crate :: SyntaxKind :: IMPORT_KW } ; [ implements ] => { $ crate :: SyntaxKind :: IMPLEMENTS_KW } ; [ new ] => { $ crate :: SyntaxKind :: NEW_KW } ; [ null ] => { $ crate :: SyntaxKind :: NULL_KW } ; [ package ] => { $ crate :: SyntaxKind :: PACKAGE_KW } ; [ private ] => { $ crate :: SyntaxKind :: PRIVATE_KW } ; [ protected ] => { $ crate :: SyntaxKind :: PROTECTED_KW } ; [ public ] => { $ crate :: SyntaxKind :: PUBLIC_KW } ; [ return ] => { $ crate :: SyntaxKind :: RETURN_KW } ; [ super ] => { $ crate :: SyntaxKind :: SUPER_KW } ; [ switch ] => { $ crate :: SyntaxKind :: SWITCH_KW } ; [ this ] => { $ crate :: SyntaxKind :: THIS_KW } ; [ throw ] => { $ crate :: SyntaxKind :: THROW_KW } ; [ try ] => { $ crate :: SyntaxKind :: TRY_KW } ; [ true ] => { $ crate :: SyntaxKind :: TRUE_KW } ; [ typeof ] => { $ crate :: SyntaxKind :: TYPEOF_KW } ; [ var ] => { $ crate :: SyntaxKind :: VAR_KW } ; [ void ] => { $ crate :: SyntaxKind :: VOID_KW } ; [ while ] => { $ crate :: SyntaxKind :: WHILE_KW } ; [ with ] => { $ crate :: SyntaxKind :: WITH_KW } ; [ yield ] => { $ crate :: SyntaxKind :: YIELD_KW } ; [ ident ] => { $ crate :: SyntaxKind :: IDENT } ; }
+macro_rules ! T { [ ; ] => { $ crate :: SyntaxKind :: SEMICOLON } ; [ , ] => { $ crate :: SyntaxKind :: COMMA } ; [ '(' ] => { $ crate :: SyntaxKind :: L_PAREN } ; [ ')' ] => { $ crate :: SyntaxKind :: R_PAREN } ; [ '{' ] => { $ crate :: SyntaxKind :: L_CURLY } ; [ '}' ] => { $ crate :: SyntaxKind :: R_CURLY } ; [ '[' ] => { $ crate :: SyntaxKind :: L_BRACK } ; [ ']' ] => { $ crate :: SyntaxKind :: R_BRACK } ; [ < ] => { $ crate :: SyntaxKind :: L_ANGLE } ; [ > ] => { $ crate :: SyntaxKind :: R_ANGLE } ; [ ~ ] => { $ crate :: SyntaxKind :: TILDE } ; [ ? ] => { $ crate :: SyntaxKind :: QUESTION } ; [ ?? ] => { $ crate :: SyntaxKind :: QUESTION2 } ; [ ?. ] => { $ crate :: SyntaxKind :: QUESTIONDOT } ; [ & ] => { $ crate :: SyntaxKind :: AMP } ; [ | ] => { $ crate :: SyntaxKind :: PIPE } ; [ + ] => { $ crate :: SyntaxKind :: PLUS } ; [ ++ ] => { $ crate :: SyntaxKind :: PLUS2 } ; [ * ] => { $ crate :: SyntaxKind :: STAR } ; [ ** ] => { $ crate :: SyntaxKind :: STAR2 } ; [ / ] => { $ crate :: SyntaxKind :: SLASH } ; [ ^ ] => { $ crate :: SyntaxKind :: CARET } ; [ % ] => { $ crate :: SyntaxKind :: PERCENT } ; [ . ] => { $ crate :: SyntaxKind :: DOT } ; [ ... ] => { $ crate :: SyntaxKind :: DOT2 } ; [ : ] => { $ crate :: SyntaxKind :: COLON } ; [ = ] => { $ crate :: SyntaxKind :: EQ } ; [ == ] => { $ crate :: SyntaxKind :: EQ2 } ; [ === ] => { $ crate :: SyntaxKind :: EQ3 } ; [ => ] => { $ crate :: SyntaxKind :: FAT_ARROW } ; [ ! ] => { $ crate :: SyntaxKind :: BANG } ; [ != ] => { $ crate :: SyntaxKind :: NEQ } ; [ !== ] => { $ crate :: SyntaxKind :: NEQ2 } ; [ - ] => { $ crate :: SyntaxKind :: MINUS } ; [ -- ] => { $ crate :: SyntaxKind :: MINUS2 } ; [ <= ] => { $ crate :: SyntaxKind :: LTEQ } ; [ >= ] => { $ crate :: SyntaxKind :: GTEQ } ; [ += ] => { $ crate :: SyntaxKind :: PLUSEQ } ; [ -= ] => { $ crate :: SyntaxKind :: MINUSEQ } ; [ |= ] => { $ crate :: SyntaxKind :: PIPEEQ } ; [ &= ] => { $ crate :: SyntaxKind :: AMPEQ } ; [ ^= ] => { $ crate :: SyntaxKind :: CARETEQ } ; [ /= ] => { $ crate :: SyntaxKind :: SLASHEQ } ; [ *= ] => { $ crate :: SyntaxKind :: STAREQ } ; [ %= ] => { $ crate :: SyntaxKind :: PERCENTEQ } ; [ && ] => { $ crate :: SyntaxKind :: AMP2 } ; [ || ] => { $ crate :: SyntaxKind :: PIPE2 } ; [ << ] => { $ crate :: SyntaxKind :: SHL } ; [ >> ] => { $ crate :: SyntaxKind :: SHR } ; [ >>> ] => { $ crate :: SyntaxKind :: USHR } ; [ <<= ] => { $ crate :: SyntaxKind :: SHLEQ } ; [ >>= ] => { $ crate :: SyntaxKind :: SHREQ } ; [ >>>= ] => { $ crate :: SyntaxKind :: USHREQ } ; [ &&= ] => { $ crate :: SyntaxKind :: AMP2EQ } ; [ ||= ] => { $ crate :: SyntaxKind :: PIPE2EQ } ; [ **= ] => { $ crate :: SyntaxKind :: STAR2EQ } ; [ ??= ] => { $ crate :: SyntaxKind :: QUESTION2EQ } ; [ await ] => { $ crate :: SyntaxKind :: AWAIT_KW } ; [ break ] => { $ crate :: SyntaxKind :: BREAK_KW } ; [ case ] => { $ crate :: SyntaxKind :: CASE_KW } ; [ catch ] => { $ crate :: SyntaxKind :: CATCH_KW } ; [ class ] => { $ crate :: SyntaxKind :: CLASS_KW } ; [ const ] => { $ crate :: SyntaxKind :: CONST_KW } ; [ continue ] => { $ crate :: SyntaxKind :: CONTINUE_KW } ; [ debugger ] => { $ crate :: SyntaxKind :: DEBUGGER_KW } ; [ default ] => { $ crate :: SyntaxKind :: DEFAULT_KW } ; [ delete ] => { $ crate :: SyntaxKind :: DELETE_KW } ; [ do ] => { $ crate :: SyntaxKind :: DO_KW } ; [ else ] => { $ crate :: SyntaxKind :: ELSE_KW } ; [ enum ] => { $ crate :: SyntaxKind :: ENUM_KW } ; [ export ] => { $ crate :: SyntaxKind :: EXPORT_KW } ; [ extends ] => { $ crate :: SyntaxKind :: EXTENDS_KW } ; [ false ] => { $ crate :: SyntaxKind :: FALSE_KW } ; [ finally ] => { $ crate :: SyntaxKind :: FINALLY_KW } ; [ for ] => { $ crate :: SyntaxKind :: FOR_KW } ; [ function ] => { $ crate :: SyntaxKind :: FUNCTION_KW } ; [ if ] => { $ crate :: SyntaxKind :: IF_KW } ; [ in ] => { $ crate :: SyntaxKind :: IN_KW } ; [ instanceof ] => { $ crate :: SyntaxKind :: INSTANCEOF_KW } ; [ interface ] => { $ crate :: SyntaxKind :: INTERFACE_KW } ; [ import ] => { $ crate :: SyntaxKind :: IMPORT_KW } ; [ implements ] => { $ crate :: SyntaxKind :: IMPLEMENTS_KW } ; [ new ] => { $ crate :: SyntaxKind :: NEW_KW } ; [ null ] => { $ crate :: SyntaxKind :: NULL_KW } ; [ package ] => { $ crate :: SyntaxKind :: PACKAGE_KW } ; [ private ] => { $ crate :: SyntaxKind :: PRIVATE_KW } ; [ protected ] => { $ crate :: SyntaxKind :: PROTECTED_KW } ; [ public ] => { $ crate :: SyntaxKind :: PUBLIC_KW } ; [ return ] => { $ crate :: SyntaxKind :: RETURN_KW } ; [ super ] => { $ crate :: SyntaxKind :: SUPER_KW } ; [ switch ] => { $ crate :: SyntaxKind :: SWITCH_KW } ; [ this ] => { $ crate :: SyntaxKind :: THIS_KW } ; [ throw ] => { $ crate :: SyntaxKind :: THROW_KW } ; [ try ] => { $ crate :: SyntaxKind :: TRY_KW } ; [ true ] => { $ crate :: SyntaxKind :: TRUE_KW } ; [ typeof ] => { $ crate :: SyntaxKind :: TYPEOF_KW } ; [ var ] => { $ crate :: SyntaxKind :: VAR_KW } ; [ void ] => { $ crate :: SyntaxKind :: VOID_KW } ; [ while ] => { $ crate :: SyntaxKind :: WHILE_KW } ; [ with ] => { $ crate :: SyntaxKind :: WITH_KW } ; [ yield ] => { $ crate :: SyntaxKind :: YIELD_KW } ; [ ident ] => { $ crate :: SyntaxKind :: IDENT } ; [ shebang ] => { $ crate :: SyntaxKind :: SHEBANG } ; }
diff --git a/xtask/src/ast.rs b/xtask/src/ast.rs
@@ -306,10 +306,12 @@ pub(crate) const AST_SRC: AstSrc = AstSrc {
     tokens: &["Whitespace", "Comment", "String"],
     nodes: &ast_nodes! {
         struct Script {
+            T![shebang],
             items: [Stmt],
         }
 
         struct Module {
+            T![shebang],
             items: [ModuleItem],
         }
 

diff --git a/xtask/src/codegen/syntax.rs b/xtask/src/codegen/syntax.rs
@@ -365,6 +365,7 @@ fn generate_syntax_kinds(grammar: KindsSrc<'_>) -> Result<String> {
             #([#punctuation_values] => { $crate::SyntaxKind::#punctuation };)*
             #([#all_keywords_idents] => { $crate::SyntaxKind::#all_keywords };)*
             [ident] => { $crate::SyntaxKind::IDENT };
+            [shebang] => { $crate::SyntaxKind::SHEBANG };
         }
     };