challenge(formatter): (also parser) Respect, parse, and format BOM ch…

…aracters from source files (#750)

Cargo.toml

@@ -89,7 +89,8 @@ serde_json        = "1.0.96"
 smallvec          = { version = "1.10.0", features = ["union", "const_new"] }
 tracing           = { version = "0.1.37", default-features = false, features = ["std"] }
 # pinning to version 1.18 to avoid multiple versions of windows-sys as dependency
-tokio = { version = "~1.18.5" }
+tokio       = { version = "~1.18.5" }
+unicode-bom = "2.0.3"
 
 
 [profile.dev.package.biome_wasm]

crates/biome_css_parser/Cargo.toml

@@ -19,6 +19,7 @@ biome_js_unicode_table = { workspace = true }
 biome_parser           = { workspace = true }
 biome_rowan            = { workspace = true }
 tracing                = { workspace = true }
+unicode-bom            = { workspace = true }
 
 [dev-dependencies]
 biome_test_utils  = { path = "../biome_test_utils" }

crates/biome_css_parser/src/lexer/mod.rs

@@ -7,6 +7,7 @@ use biome_js_unicode_table::{is_id_continue, is_id_start, lookup_byte, Dispatch:
 use biome_parser::diagnostic::ParseDiagnostic;
 use biome_parser::lexer::{LexContext, Lexer, LexerCheckpoint, TokenFlags};
 use std::char::REPLACEMENT_CHARACTER;
+use unicode_bom::Bom;
 
 #[derive(Debug, Copy, Clone, Eq, PartialEq, Default)]
 pub enum CssLexContext {
@@ -41,6 +42,9 @@ pub(crate) struct CssLexer<'src> {
     /// `true` if there has been a line break between the last non-trivia token and the next non-trivia token.
     after_newline: bool,
 
+    /// If the source starts with a Unicode BOM, this is the number of bytes for that token.
+    unicode_bom_length: usize,
+
     /// Byte offset of the current token from the start of the source
     /// The range of the current token can be computed by `self.position - self.current_start`
     current_start: TextSize,
@@ -78,6 +82,7 @@ impl<'src> Lexer<'src> for CssLexer<'src> {
             current_flags: self.current_flags,
             current_kind: self.current_kind,
             after_line_break: self.after_newline,
+            unicode_bom_length: self.unicode_bom_length,
             diagnostics_pos: self.diagnostics.len() as u32,
         }
     }
@@ -139,6 +144,7 @@ impl<'src> Lexer<'src> for CssLexer<'src> {
             current_flags,
             current_kind,
             after_line_break,
+            unicode_bom_length,
             diagnostics_pos,
         } = checkpoint;
 
@@ -149,6 +155,7 @@ impl<'src> Lexer<'src> for CssLexer<'src> {
         self.current_start = current_start;
         self.current_flags = current_flags;
         self.after_newline = after_line_break;
+        self.unicode_bom_length = unicode_bom_length;
         self.diagnostics.truncate(diagnostics_pos as usize);
     }
 
@@ -167,6 +174,7 @@ impl<'src> CssLexer<'src> {
         Self {
             source,
             after_newline: false,
+            unicode_bom_length: 0,
             current_kind: TOMBSTONE,
             current_start: TextSize::from(0),
             current_flags: TokenFlags::empty(),
@@ -259,6 +267,31 @@ impl<'src> CssLexer<'src> {
         }
     }
 
+    /// Check if the source starts with a Unicode BOM character. If it does,
+    /// consume it and return the UNICODE_BOM token kind.
+    ///
+    /// ## Safety
+    /// Must be called at a valid UT8 char boundary (and realistically only at
+    /// the start position of the source).
+    fn consume_potential_bom(&mut self) -> Option<CssSyntaxKind> {
+        // Bom needs at least the first three bytes of the source to know if it
+        // matches the UTF-8 BOM and not an alternative. This can be expanded
+        // to more bytes to support other BOM characters if Biome decides to
+        // support other encodings like UTF-16.
+        if let Some(first) = self.source().get(0..3) {
+            let bom = Bom::from(first.as_bytes());
+            self.unicode_bom_length = bom.len();
+            self.advance(self.unicode_bom_length);
+
+            match bom {
+                Bom::Null => None,
+                _ => Some(UNICODE_BOM),
+            }
+        } else {
+            None
+        }
+    }
+
     /// Get the UTF8 char which starts at the current byte
     ///
     /// ## Safety
@@ -451,6 +484,16 @@ impl<'src> CssLexer<'src> {
             PIP => self.consume_pipe(),
             EQL => self.eat_byte(T![=]),
 
+            UNI => {
+                // A BOM can only appear at the start of a file, so if we haven't advanced at all yet,
+                // perform the check. At any other position, the BOM is just considered plain whitespace.
+                if self.position == 0 && self.consume_potential_bom().is_some() {
+                    UNICODE_BOM
+                } else {
+                    self.eat_unexpected_character()
+                }
+            }
+
             _ => self.eat_unexpected_character(),
         }
     }

crates/biome_css_parser/src/syntax/mod.rs

@@ -21,6 +21,7 @@ const BODY_RECOVERY_SET: TokenSet<CssSyntaxKind> =
 
 pub(crate) fn parse_root(p: &mut CssParser) {
     let m = p.start();
+    p.eat(UNICODE_BOM);
 
     parse_rule_list(p);
 

crates/biome_css_parser/tests/css_test_suite/error/css_unfinished_block.css.snap

@@ -15,6 +15,7 @@ expression: snapshot
 
 ```
 CssRoot {
+    bom_token: missing (optional),
     rules: CssRuleList [
         CssRule {
             prelude: CssSelectorList [
@@ -46,7 +47,8 @@ CssRoot {
 
 ```
 0: CSS_ROOT@0..10
-  0: CSS_RULE_LIST@0..9
+  0: (empty)
+  1: CSS_RULE_LIST@0..9
     0: CSS_RULE@0..9
       0: CSS_SELECTOR_LIST@0..8
         0: CSS_COMPOUND_SELECTOR@0..8
@@ -61,7 +63,7 @@ CssRoot {
         0: L_CURLY@8..9 "{" [] []
         1: CSS_DECLARATION_LIST@9..9
         2: (empty)
-  1: EOF@9..10 "" [Newline("\n")] []
+  2: EOF@9..10 "" [Newline("\n")] []
 
 ```
 

crates/biome_css_parser/tests/css_test_suite/error/selector/attribute_selector.css.snap

@@ -20,6 +20,7 @@ expression: snapshot
 
 ```
 CssRoot {
+    bom_token: missing (optional),
     rules: CssRuleList [
         CssRule {
             prelude: CssSelectorList [
@@ -150,7 +151,8 @@ CssRoot {
 
 ```
 0: CSS_ROOT@0..66
-  0: CSS_RULE_LIST@0..65
+  0: (empty)
+  1: CSS_RULE_LIST@0..65
     0: CSS_RULE@0..11
       0: CSS_SELECTOR_LIST@0..9
         0: CSS_COMPOUND_SELECTOR@0..9
@@ -225,7 +227,7 @@ CssRoot {
                     0: CSS_STRING_LITERAL@58..64 "\"foo\"" [] [Whitespace(" ")]
                 2: S_KW@64..65 "s" [] []
               3: (empty)
-  1: EOF@65..66 "" [Newline("\n")] []
+  2: EOF@65..66 "" [Newline("\n")] []
 
 ```
 

crates/biome_css_parser/tests/css_test_suite/error/selector/class_selector_err.css.snap

@@ -15,6 +15,7 @@ expression: snapshot
 
 ```
 CssRoot {
+    bom_token: missing (optional),
     rules: CssRuleList [
         CssRule {
             prelude: CssSelectorList [
@@ -50,7 +51,8 @@ CssRoot {
 
 ```
 0: CSS_ROOT@0..12
-  0: CSS_RULE_LIST@0..11
+  0: (empty)
+  1: CSS_RULE_LIST@0..11
     0: CSS_RULE@0..11
       0: CSS_SELECTOR_LIST@0..9
         0: CSS_COMPOUND_SELECTOR@0..9
@@ -68,7 +70,7 @@ CssRoot {
         0: L_CURLY@9..10 "{" [] []
         1: CSS_DECLARATION_LIST@10..10
         2: R_CURLY@10..11 "}" [] []
-  1: EOF@11..12 "" [Newline("\n")] []
+  2: EOF@11..12 "" [Newline("\n")] []
 
 ```
 

crates/biome_css_parser/tests/css_test_suite/error/selector/id_selector_err.css.snap

@@ -15,6 +15,7 @@ expression: snapshot
 
 ```
 CssRoot {
+    bom_token: missing (optional),
     rules: CssRuleList [
         CssRule {
             prelude: CssSelectorList [
@@ -50,7 +51,8 @@ CssRoot {
 
 ```
 0: CSS_ROOT@0..12
-  0: CSS_RULE_LIST@0..11
+  0: (empty)
+  1: CSS_RULE_LIST@0..11
     0: CSS_RULE@0..11
       0: CSS_SELECTOR_LIST@0..9
         0: CSS_COMPOUND_SELECTOR@0..9
@@ -68,7 +70,7 @@ CssRoot {
         0: L_CURLY@9..10 "{" [] []
         1: CSS_DECLARATION_LIST@10..10
         2: R_CURLY@10..11 "}" [] []
-  1: EOF@11..12 "" [Newline("\n")] []
+  2: EOF@11..12 "" [Newline("\n")] []
 
 ```
 

crates/biome_css_parser/tests/css_test_suite/error/selector/invalid_selector.css.snap

@@ -17,6 +17,7 @@ expression: snapshot
 
 ```
 CssRoot {
+    bom_token: missing (optional),
     rules: CssRuleList [
         CssRule {
             prelude: CssSelectorList [
@@ -54,7 +55,8 @@ CssRoot {
 
 ```
 0: CSS_ROOT@0..16
-  0: CSS_RULE_LIST@0..15
+  0: (empty)
+  1: CSS_RULE_LIST@0..15
     0: CSS_RULE@0..15
       0: CSS_SELECTOR_LIST@0..11
         0: CSS_BOGUS_SELECTOR@0..1
@@ -72,7 +74,7 @@ CssRoot {
         0: L_CURLY@11..12 "{" [] []
         1: CSS_DECLARATION_LIST@12..12
         2: R_CURLY@12..15 "}" [Newline("\n"), Newline("\n")] []
-  1: EOF@15..16 "" [Newline("\n")] []
+  2: EOF@15..16 "" [Newline("\n")] []
 
 ```
 

crates/biome_css_parser/tests/css_test_suite/error/selector/missing_identifier.css.snap

@@ -15,6 +15,7 @@ expression: snapshot
 
 ```
 CssRoot {
+    bom_token: missing (optional),
     rules: CssRuleList [
         CssRule {
             prelude: CssSelectorList [
@@ -55,7 +56,8 @@ CssRoot {
 
 ```
 0: CSS_ROOT@0..9
-  0: CSS_RULE_LIST@0..8
+  0: (empty)
+  1: CSS_RULE_LIST@0..8
     0: CSS_RULE@0..8
       0: CSS_SELECTOR_LIST@0..6
         0: CSS_COMPOUND_SELECTOR@0..2
@@ -77,7 +79,7 @@ CssRoot {
         0: L_CURLY@6..7 "{" [] []
         1: CSS_DECLARATION_LIST@7..7
         2: R_CURLY@7..8 "}" [] []
-  1: EOF@8..9 "" [Newline("\n")] []
+  2: EOF@8..9 "" [Newline("\n")] []
 
 ```