Feat: add shebang handling to lexer and parser

crates/rslint_lexer/CHANGELOG.md

@@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased]
 
+### Added
+
+- Added handling of shebang
+
 ### Fixed
 
 - Fixed handling of `/=` and `>>=`

crates/rslint_lexer/src/lib.rs

@@ -757,6 +757,32 @@ impl<'src> Lexer<'src> {
         }
     }
 
+    #[inline]
+    fn read_shebang(&mut self) -> LexerReturn {
+        assert_eq!(self.cur, 0);
+        self.next();
+        if let Some(b"#!") = self.bytes.get(0..2) {
+            while self.next().is_some() {
+                let chr = self.get_unicode_char();
+                self.cur += chr.len_utf8() - 1;
+
+                if is_linebreak(chr) {
+                    break;
+                }
+            }
+            tok!(SHEBANG, self.cur)
+        } else {
+            let err = Diagnostic::error()
+                .with_message("Incomplete shebang")
+                .with_labels(vec![
+                    Label::primary(self.file_id, 0..1)
+                        .with_message("Hash is located at the beginning of the file, which is supposed to be interpreted as shebang, but following `!` is not found")
+                ]);
+
+            (Token::new(SyntaxKind::SHEBANG, 1), Some(err))
+        }
+    }
+
     #[inline]
     fn read_slash(&mut self) -> LexerReturn {
         let start = self.cur;
@@ -1130,6 +1156,14 @@ impl<'src> Lexer<'src> {
                 tok!(WHITESPACE, self.cur - start)
             }
             EXL => self.resolve_bang(),
+            HAS => {
+                if self.cur == 0 {
+                    self.read_shebang()
+                } else {
+                    self.consume_ident();
+                    tok!(IDENT, self.cur - start)
+                }
+            }
             PRC => self.bin_or_assign(T![%], T![%=]),
             AMP => self.resolve_amp(),
             PNO => self.eat(tok!(L_PAREN, 1)),
@@ -1366,6 +1400,7 @@ enum Dispatch {
     EXL,
     QOT,
     IDT,
+    HAS,
     PRC,
     AMP,
     PNO,
@@ -1418,7 +1453,7 @@ static DISPATCHER: [Dispatch; 256] = [
     //   0    1    2    3    4    5    6    7    8    9    A    B    C    D    E    F   //
     ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, WHS, WHS, WHS, WHS, WHS, ERR, ERR, // 0
     ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, ERR, // 1
-    WHS, EXL, QOT, ERR, IDT, PRC, AMP, QOT, PNO, PNC, MUL, PLS, COM, MIN, PRD, SLH, // 2
+    WHS, EXL, QOT, HAS, IDT, PRC, AMP, QOT, PNO, PNC, MUL, PLS, COM, MIN, PRD, SLH, // 2
     ZER, DIG, DIG, DIG, DIG, DIG, DIG, DIG, DIG, DIG, COL, SEM, LSS, EQL, MOR, QST, // 3
     ERR, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, // 4
     IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, IDT, BTO, BSL, BTC, CRT, IDT, // 5

crates/rslint_lexer/src/tests.rs

@@ -89,6 +89,19 @@ fn strip_shebang() {
     assert_eq!(lex.cur, 13);
 }
 
+#[test]
+fn shebang() {
+    assert_lex! {
+        "#! /bin/node",
+        SHEBANG:12
+    }
+    assert_lex! {
+        "#!/bin/node\n",
+        SHEBANG:11,
+        WHITESPACE:1
+    }
+}
+
 #[test]
 fn empty() {
     assert_lex! {

crates/rslint_parser/CHANGELOG.MD

@@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased]
 
+### Added
+
+- Added handling of shebang where the parser consumes shebang if exists at the start
+
 ### Fixed
 
 - Fixed handling of `/=` and `>>=`

crates/rslint_parser/src/ast/generated/nodes.rs

@@ -10,13 +10,15 @@ pub struct Script {
     pub(crate) syntax: SyntaxNode,
 }
 impl Script {
+    pub fn shebang_token(&self) -> Option<SyntaxToken> { support::token(&self.syntax, T![shebang]) }
     pub fn items(&self) -> AstChildren<Stmt> { support::children(&self.syntax) }
 }
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct Module {
     pub(crate) syntax: SyntaxNode,
 }
 impl Module {
+    pub fn shebang_token(&self) -> Option<SyntaxToken> { support::token(&self.syntax, T![shebang]) }
     pub fn items(&self) -> AstChildren<ModuleItem> { support::children(&self.syntax) }
 }
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]

crates/rslint_parser/src/syntax/program.rs

@@ -16,6 +16,7 @@ pub fn script(p: &mut Parser) -> CompletedMarker {
         "Using the script parsing function for modules is erroneous"
     );
     let m = p.start();
+    p.eat(T![shebang]);
     block_items(p, true, true, None);
     m.complete(p, SyntaxKind::SCRIPT)
 }
@@ -30,6 +31,7 @@ pub fn module(p: &mut Parser) -> CompletedMarker {
         "Using the module parsing function for scripts is erroneous"
     );
     let m = p.start();
+    p.eat(T![shebang]);
     block_items(p, true, true, None);
     m.complete(p, SyntaxKind::MODULE)
 }

crates/rslint_syntax/src/generated.rs

@@ -401,4 +401,4 @@ impl SyntaxKind {
 }
 #[doc = r" Utility macro for creating a SyntaxKind through simple macro syntax"]
 #[macro_export]
-macro_rules ! T { [ ; ] => { $ crate :: SyntaxKind :: SEMICOLON } ; [ , ] => { $ crate :: SyntaxKind :: COMMA } ; [ '(' ] => { $ crate :: SyntaxKind :: L_PAREN } ; [ ')' ] => { $ crate :: SyntaxKind :: R_PAREN } ; [ '{' ] => { $ crate :: SyntaxKind :: L_CURLY } ; [ '}' ] => { $ crate :: SyntaxKind :: R_CURLY } ; [ '[' ] => { $ crate :: SyntaxKind :: L_BRACK } ; [ ']' ] => { $ crate :: SyntaxKind :: R_BRACK } ; [ < ] => { $ crate :: SyntaxKind :: L_ANGLE } ; [ > ] => { $ crate :: SyntaxKind :: R_ANGLE } ; [ ~ ] => { $ crate :: SyntaxKind :: TILDE } ; [ ? ] => { $ crate :: SyntaxKind :: QUESTION } ; [ ?? ] => { $ crate :: SyntaxKind :: QUESTION2 } ; [ ?. ] => { $ crate :: SyntaxKind :: QUESTIONDOT } ; [ & ] => { $ crate :: SyntaxKind :: AMP } ; [ | ] => { $ crate :: SyntaxKind :: PIPE } ; [ + ] => { $ crate :: SyntaxKind :: PLUS } ; [ ++ ] => { $ crate :: SyntaxKind :: PLUS2 } ; [ * ] => { $ crate :: SyntaxKind :: STAR } ; [ ** ] => { $ crate :: SyntaxKind :: STAR2 } ; [ / ] => { $ crate :: SyntaxKind :: SLASH } ; [ ^ ] => { $ crate :: SyntaxKind :: CARET } ; [ % ] => { $ crate :: SyntaxKind :: PERCENT } ; [ . ] => { $ crate :: SyntaxKind :: DOT } ; [ ... ] => { $ crate :: SyntaxKind :: DOT2 } ; [ : ] => { $ crate :: SyntaxKind :: COLON } ; [ = ] => { $ crate :: SyntaxKind :: EQ } ; [ == ] => { $ crate :: SyntaxKind :: EQ2 } ; [ === ] => { $ crate :: SyntaxKind :: EQ3 } ; [ => ] => { $ crate :: SyntaxKind :: FAT_ARROW } ; [ ! ] => { $ crate :: SyntaxKind :: BANG } ; [ != ] => { $ crate :: SyntaxKind :: NEQ } ; [ !== ] => { $ crate :: SyntaxKind :: NEQ2 } ; [ - ] => { $ crate :: SyntaxKind :: MINUS } ; [ -- ] => { $ crate :: SyntaxKind :: MINUS2 } ; [ <= ] => { $ crate :: SyntaxKind :: LTEQ } ; [ >= ] => { $ crate :: SyntaxKind :: GTEQ } ; [ += ] => { $ crate :: SyntaxKind :: PLUSEQ } ; [ -= ] => { $ crate :: SyntaxKind :: MINUSEQ } ; [ |= ] => { $ crate :: SyntaxKind :: PIPEEQ } ; [ &= ] => { $ crate :: SyntaxKind :: AMPEQ } ; [ ^= ] => { $ crate :: SyntaxKind :: CARETEQ } ; [ /= ] => { $ crate :: SyntaxKind :: SLASHEQ } ; [ *= ] => { $ crate :: SyntaxKind :: STAREQ } ; [ %= ] => { $ crate :: SyntaxKind :: PERCENTEQ } ; [ && ] => { $ crate :: SyntaxKind :: AMP2 } ; [ || ] => { $ crate :: SyntaxKind :: PIPE2 } ; [ << ] => { $ crate :: SyntaxKind :: SHL } ; [ >> ] => { $ crate :: SyntaxKind :: SHR } ; [ >>> ] => { $ crate :: SyntaxKind :: USHR } ; [ <<= ] => { $ crate :: SyntaxKind :: SHLEQ } ; [ >>= ] => { $ crate :: SyntaxKind :: SHREQ } ; [ >>>= ] => { $ crate :: SyntaxKind :: USHREQ } ; [ &&= ] => { $ crate :: SyntaxKind :: AMP2EQ } ; [ ||= ] => { $ crate :: SyntaxKind :: PIPE2EQ } ; [ **= ] => { $ crate :: SyntaxKind :: STAR2EQ } ; [ ??= ] => { $ crate :: SyntaxKind :: QUESTION2EQ } ; [ await ] => { $ crate :: SyntaxKind :: AWAIT_KW } ; [ break ] => { $ crate :: SyntaxKind :: BREAK_KW } ; [ case ] => { $ crate :: SyntaxKind :: CASE_KW } ; [ catch ] => { $ crate :: SyntaxKind :: CATCH_KW } ; [ class ] => { $ crate :: SyntaxKind :: CLASS_KW } ; [ const ] => { $ crate :: SyntaxKind :: CONST_KW } ; [ continue ] => { $ crate :: SyntaxKind :: CONTINUE_KW } ; [ debugger ] => { $ crate :: SyntaxKind :: DEBUGGER_KW } ; [ default ] => { $ crate :: SyntaxKind :: DEFAULT_KW } ; [ delete ] => { $ crate :: SyntaxKind :: DELETE_KW } ; [ do ] => { $ crate :: SyntaxKind :: DO_KW } ; [ else ] => { $ crate :: SyntaxKind :: ELSE_KW } ; [ enum ] => { $ crate :: SyntaxKind :: ENUM_KW } ; [ export ] => { $ crate :: SyntaxKind :: EXPORT_KW } ; [ extends ] => { $ crate :: SyntaxKind :: EXTENDS_KW } ; [ false ] => { $ crate :: SyntaxKind :: FALSE_KW } ; [ finally ] => { $ crate :: SyntaxKind :: FINALLY_KW } ; [ for ] => { $ crate :: SyntaxKind :: FOR_KW } ; [ function ] => { $ crate :: SyntaxKind :: FUNCTION_KW } ; [ if ] => { $ crate :: SyntaxKind :: IF_KW } ; [ in ] => { $ crate :: SyntaxKind :: IN_KW } ; [ instanceof ] => { $ crate :: SyntaxKind :: INSTANCEOF_KW } ; [ interface ] => { $ crate :: SyntaxKind :: INTERFACE_KW } ; [ import ] => { $ crate :: SyntaxKind :: IMPORT_KW } ; [ implements ] => { $ crate :: SyntaxKind :: IMPLEMENTS_KW } ; [ new ] => { $ crate :: SyntaxKind :: NEW_KW } ; [ null ] => { $ crate :: SyntaxKind :: NULL_KW } ; [ package ] => { $ crate :: SyntaxKind :: PACKAGE_KW } ; [ private ] => { $ crate :: SyntaxKind :: PRIVATE_KW } ; [ protected ] => { $ crate :: SyntaxKind :: PROTECTED_KW } ; [ public ] => { $ crate :: SyntaxKind :: PUBLIC_KW } ; [ return ] => { $ crate :: SyntaxKind :: RETURN_KW } ; [ super ] => { $ crate :: SyntaxKind :: SUPER_KW } ; [ switch ] => { $ crate :: SyntaxKind :: SWITCH_KW } ; [ this ] => { $ crate :: SyntaxKind :: THIS_KW } ; [ throw ] => { $ crate :: SyntaxKind :: THROW_KW } ; [ try ] => { $ crate :: SyntaxKind :: TRY_KW } ; [ true ] => { $ crate :: SyntaxKind :: TRUE_KW } ; [ typeof ] => { $ crate :: SyntaxKind :: TYPEOF_KW } ; [ var ] => { $ crate :: SyntaxKind :: VAR_KW } ; [ void ] => { $ crate :: SyntaxKind :: VOID_KW } ; [ while ] => { $ crate :: SyntaxKind :: WHILE_KW } ; [ with ] => { $ crate :: SyntaxKind :: WITH_KW } ; [ yield ] => { $ crate :: SyntaxKind :: YIELD_KW } ; [ ident ] => { $ crate :: SyntaxKind :: IDENT } ; }
+macro_rules ! T { [ ; ] => { $ crate :: SyntaxKind :: SEMICOLON } ; [ , ] => { $ crate :: SyntaxKind :: COMMA } ; [ '(' ] => { $ crate :: SyntaxKind :: L_PAREN } ; [ ')' ] => { $ crate :: SyntaxKind :: R_PAREN } ; [ '{' ] => { $ crate :: SyntaxKind :: L_CURLY } ; [ '}' ] => { $ crate :: SyntaxKind :: R_CURLY } ; [ '[' ] => { $ crate :: SyntaxKind :: L_BRACK } ; [ ']' ] => { $ crate :: SyntaxKind :: R_BRACK } ; [ < ] => { $ crate :: SyntaxKind :: L_ANGLE } ; [ > ] => { $ crate :: SyntaxKind :: R_ANGLE } ; [ ~ ] => { $ crate :: SyntaxKind :: TILDE } ; [ ? ] => { $ crate :: SyntaxKind :: QUESTION } ; [ ?? ] => { $ crate :: SyntaxKind :: QUESTION2 } ; [ ?. ] => { $ crate :: SyntaxKind :: QUESTIONDOT } ; [ & ] => { $ crate :: SyntaxKind :: AMP } ; [ | ] => { $ crate :: SyntaxKind :: PIPE } ; [ + ] => { $ crate :: SyntaxKind :: PLUS } ; [ ++ ] => { $ crate :: SyntaxKind :: PLUS2 } ; [ * ] => { $ crate :: SyntaxKind :: STAR } ; [ ** ] => { $ crate :: SyntaxKind :: STAR2 } ; [ / ] => { $ crate :: SyntaxKind :: SLASH } ; [ ^ ] => { $ crate :: SyntaxKind :: CARET } ; [ % ] => { $ crate :: SyntaxKind :: PERCENT } ; [ . ] => { $ crate :: SyntaxKind :: DOT } ; [ ... ] => { $ crate :: SyntaxKind :: DOT2 } ; [ : ] => { $ crate :: SyntaxKind :: COLON } ; [ = ] => { $ crate :: SyntaxKind :: EQ } ; [ == ] => { $ crate :: SyntaxKind :: EQ2 } ; [ === ] => { $ crate :: SyntaxKind :: EQ3 } ; [ => ] => { $ crate :: SyntaxKind :: FAT_ARROW } ; [ ! ] => { $ crate :: SyntaxKind :: BANG } ; [ != ] => { $ crate :: SyntaxKind :: NEQ } ; [ !== ] => { $ crate :: SyntaxKind :: NEQ2 } ; [ - ] => { $ crate :: SyntaxKind :: MINUS } ; [ -- ] => { $ crate :: SyntaxKind :: MINUS2 } ; [ <= ] => { $ crate :: SyntaxKind :: LTEQ } ; [ >= ] => { $ crate :: SyntaxKind :: GTEQ } ; [ += ] => { $ crate :: SyntaxKind :: PLUSEQ } ; [ -= ] => { $ crate :: SyntaxKind :: MINUSEQ } ; [ |= ] => { $ crate :: SyntaxKind :: PIPEEQ } ; [ &= ] => { $ crate :: SyntaxKind :: AMPEQ } ; [ ^= ] => { $ crate :: SyntaxKind :: CARETEQ } ; [ /= ] => { $ crate :: SyntaxKind :: SLASHEQ } ; [ *= ] => { $ crate :: SyntaxKind :: STAREQ } ; [ %= ] => { $ crate :: SyntaxKind :: PERCENTEQ } ; [ && ] => { $ crate :: SyntaxKind :: AMP2 } ; [ || ] => { $ crate :: SyntaxKind :: PIPE2 } ; [ << ] => { $ crate :: SyntaxKind :: SHL } ; [ >> ] => { $ crate :: SyntaxKind :: SHR } ; [ >>> ] => { $ crate :: SyntaxKind :: USHR } ; [ <<= ] => { $ crate :: SyntaxKind :: SHLEQ } ; [ >>= ] => { $ crate :: SyntaxKind :: SHREQ } ; [ >>>= ] => { $ crate :: SyntaxKind :: USHREQ } ; [ &&= ] => { $ crate :: SyntaxKind :: AMP2EQ } ; [ ||= ] => { $ crate :: SyntaxKind :: PIPE2EQ } ; [ **= ] => { $ crate :: SyntaxKind :: STAR2EQ } ; [ ??= ] => { $ crate :: SyntaxKind :: QUESTION2EQ } ; [ await ] => { $ crate :: SyntaxKind :: AWAIT_KW } ; [ break ] => { $ crate :: SyntaxKind :: BREAK_KW } ; [ case ] => { $ crate :: SyntaxKind :: CASE_KW } ; [ catch ] => { $ crate :: SyntaxKind :: CATCH_KW } ; [ class ] => { $ crate :: SyntaxKind :: CLASS_KW } ; [ const ] => { $ crate :: SyntaxKind :: CONST_KW } ; [ continue ] => { $ crate :: SyntaxKind :: CONTINUE_KW } ; [ debugger ] => { $ crate :: SyntaxKind :: DEBUGGER_KW } ; [ default ] => { $ crate :: SyntaxKind :: DEFAULT_KW } ; [ delete ] => { $ crate :: SyntaxKind :: DELETE_KW } ; [ do ] => { $ crate :: SyntaxKind :: DO_KW } ; [ else ] => { $ crate :: SyntaxKind :: ELSE_KW } ; [ enum ] => { $ crate :: SyntaxKind :: ENUM_KW } ; [ export ] => { $ crate :: SyntaxKind :: EXPORT_KW } ; [ extends ] => { $ crate :: SyntaxKind :: EXTENDS_KW } ; [ false ] => { $ crate :: SyntaxKind :: FALSE_KW } ; [ finally ] => { $ crate :: SyntaxKind :: FINALLY_KW } ; [ for ] => { $ crate :: SyntaxKind :: FOR_KW } ; [ function ] => { $ crate :: SyntaxKind :: FUNCTION_KW } ; [ if ] => { $ crate :: SyntaxKind :: IF_KW } ; [ in ] => { $ crate :: SyntaxKind :: IN_KW } ; [ instanceof ] => { $ crate :: SyntaxKind :: INSTANCEOF_KW } ; [ interface ] => { $ crate :: SyntaxKind :: INTERFACE_KW } ; [ import ] => { $ crate :: SyntaxKind :: IMPORT_KW } ; [ implements ] => { $ crate :: SyntaxKind :: IMPLEMENTS_KW } ; [ new ] => { $ crate :: SyntaxKind :: NEW_KW } ; [ null ] => { $ crate :: SyntaxKind :: NULL_KW } ; [ package ] => { $ crate :: SyntaxKind :: PACKAGE_KW } ; [ private ] => { $ crate :: SyntaxKind :: PRIVATE_KW } ; [ protected ] => { $ crate :: SyntaxKind :: PROTECTED_KW } ; [ public ] => { $ crate :: SyntaxKind :: PUBLIC_KW } ; [ return ] => { $ crate :: SyntaxKind :: RETURN_KW } ; [ super ] => { $ crate :: SyntaxKind :: SUPER_KW } ; [ switch ] => { $ crate :: SyntaxKind :: SWITCH_KW } ; [ this ] => { $ crate :: SyntaxKind :: THIS_KW } ; [ throw ] => { $ crate :: SyntaxKind :: THROW_KW } ; [ try ] => { $ crate :: SyntaxKind :: TRY_KW } ; [ true ] => { $ crate :: SyntaxKind :: TRUE_KW } ; [ typeof ] => { $ crate :: SyntaxKind :: TYPEOF_KW } ; [ var ] => { $ crate :: SyntaxKind :: VAR_KW } ; [ void ] => { $ crate :: SyntaxKind :: VOID_KW } ; [ while ] => { $ crate :: SyntaxKind :: WHILE_KW } ; [ with ] => { $ crate :: SyntaxKind :: WITH_KW } ; [ yield ] => { $ crate :: SyntaxKind :: YIELD_KW } ; [ ident ] => { $ crate :: SyntaxKind :: IDENT } ; [ shebang ] => { $ crate :: SyntaxKind :: SHEBANG } ; }

xtask/src/ast.rs

@@ -306,10 +306,12 @@ pub(crate) const AST_SRC: AstSrc = AstSrc {
     tokens: &["Whitespace", "Comment", "String"],
     nodes: &ast_nodes! {
         struct Script {
+            T![shebang],
             items: [Stmt],
         }
 
         struct Module {
+            T![shebang],
             items: [ModuleItem],
         }
 

xtask/src/codegen/syntax.rs

@@ -365,6 +365,7 @@ fn generate_syntax_kinds(grammar: KindsSrc<'_>) -> Result<String> {
             #([#punctuation_values] => { $crate::SyntaxKind::#punctuation };)*
             #([#all_keywords_idents] => { $crate::SyntaxKind::#all_keywords };)*
             [ident] => { $crate::SyntaxKind::IDENT };
+            [shebang] => { $crate::SyntaxKind::SHEBANG };
         }
     };