diff --git a/src/parse-declaration.ts b/src/parse-declaration.ts index d08122b..245a9f5 100644 --- a/src/parse-declaration.ts +++ b/src/parse-declaration.ts @@ -2,15 +2,20 @@ import { Lexer } from './tokenize' import { CSSDataArena, DECLARATION, RAW, FLAG_IMPORTANT, FLAG_BROWSERHACK } from './arena' import { ValueParser } from './parse-value' -import { is_vendor_prefixed } from './string-utils' +import { + is_vendor_prefixed, + CHAR_LEFT_BRACE, + CHAR_RIGHT_BRACE, + CHAR_SEMICOLON, + CHAR_LEFT_PAREN, + CHAR_RIGHT_PAREN, + CHAR_EXCLAMATION, +} from './string-utils' import { TOKEN_IDENT, TOKEN_COLON, TOKEN_SEMICOLON, TOKEN_DELIM, - TOKEN_EOF, - TOKEN_LEFT_BRACE, - TOKEN_RIGHT_BRACE, TOKEN_LEFT_PAREN, TOKEN_RIGHT_PAREN, TOKEN_LEFT_BRACKET, @@ -18,10 +23,9 @@ import { TOKEN_COMMA, TOKEN_HASH, TOKEN_AT_KEYWORD, - TOKEN_FUNCTION, type TokenType, } from './token-types' -import { trim_boundaries } from './parse-utils' +import { trim_boundaries, skip_whitespace_and_comments_backward } from './parse-utils' import { CSSNode } from './css-node' import type { Declaration } from './node-types' @@ -145,7 +149,11 @@ export class DeclarationParser { lexer.restore_position(has_delimiter_prefix ? initial_saved : saved) return null } - lexer.next_token_fast(true) // consume ':', skip whitespace + // Skip whitespace after ':' without tokenizing the value's first token - the raw scan + // below needs to see every value character itself (e.g. a leading "calc("'s '(' must + // reach the paren-depth tracking, not be pre-consumed here). + lexer.pos = lexer.token_end // move past ':' (already at this position, but be explicit) + lexer.skip_whitespace_in_range(this.source.length) // Create declaration node (length will be set later) let declaration = this.arena.create_node( @@ -160,47 +168,71 @@ export class DeclarationParser { this.arena.set_content_start_delta(declaration, 0) this.arena.set_content_length(declaration, prop_end - prop_start) - // Track value start (after colon, skipping whitespace) - // CRITICAL: Capture line/column for value parsing - // After consuming ':', lexer is now positioned at first value token - let value_start = lexer.token_start - let value_start_line = lexer.token_line - let value_start_column = lexer.token_column + // Value start (after colon/whitespace) - lexer is positioned at the first, untokenized char. + let value_start = lexer.pos + let value_start_line = lexer.line + let value_start_column = lexer.column let value_end = value_start // Parse value (everything until ';' or EOF) let has_important = false - let last_end = lexer.token_end + let last_end = value_start // Track parenthesis depth to handle semicolons inside functions (e.g., url(data:image/png;base64,...)) - // NOTE: Same pattern exists in parse.ts for at-rule prelude parsing - keep in sync let paren_depth = 0 - // Process tokens until we hit semicolon, EOF, or end of input - while ((lexer.token_type as TokenType) !== TOKEN_EOF && lexer.token_start < end) { - let token_type = lexer.token_type as TokenType + // Raw character scan for the value's stop points (paren depth, ';'/'}' at depth zero, + // '{', '!') - content in between doesn't need tokenizing since ValueNodeParser + // re-tokenizes the span afterward. Loop relies only on explicit break/return, since a + // paren_depth adjustment can land exactly on `end` without the "ran out" case firing. + while (true) { + let stop_ch = lexer.skip_to_declaration_stop(end) - // Track parenthesis depth - if (token_type === TOKEN_LEFT_PAREN || token_type === TOKEN_FUNCTION) { + if (stop_ch === CHAR_LEFT_PAREN) { paren_depth++ - } else if (token_type === TOKEN_RIGHT_PAREN) { + lexer.pos++ + continue + } + if (stop_ch === CHAR_RIGHT_PAREN) { paren_depth-- + lexer.pos++ + continue } - // Only break on semicolon/brace when outside all parentheses - if (token_type === TOKEN_SEMICOLON && paren_depth === 0) break - if (token_type === TOKEN_RIGHT_BRACE && paren_depth === 0) break + if (stop_ch === CHAR_SEMICOLON && paren_depth === 0) { + value_end = skip_whitespace_and_comments_backward(this.source, lexer.pos, value_start) + // Tokenize ';' so token_type reflects it, matching what the old loop left behind. + lexer.next_token_fast(false) + break + } + + if (stop_ch === CHAR_RIGHT_BRACE && paren_depth === 0) { + if (lexer.pos === value_start) { + // Degenerate case: "color:}" (no value content). Replicates a quirk of the old + // pre-tokenized scan, where '}' itself became the "first value token" and its + // end position became last_end before the per-token loop ever ran. + last_end = lexer.pos + 1 + value_end = value_start + } else { + last_end = skip_whitespace_and_comments_backward(this.source, lexer.pos, value_start) + value_end = last_end + } + // Tokenize '}' (without consuming it) so peek_type() sees it - the enclosing + // block's own loop needs to see '}' to know it's done. + lexer.next_token_fast(false) + break + } - // If we encounter '{', this is actually a style rule, not a declaration - if (token_type === TOKEN_LEFT_BRACE) { + if (stop_ch === CHAR_LEFT_BRACE) { + // Actually a style rule, not a declaration - '{' bails out at any paren depth. lexer.restore_position(saved) return null } - // Check for ! followed by any identifier (optimized: only check when we see '!') - if (token_type === TOKEN_DELIM && this.source[lexer.token_start] === '!') { + if (stop_ch === CHAR_EXCLAMATION) { // Mark end of value before !important - value_end = lexer.token_start - // Check if next token is an identifier + value_end = lexer.pos + lexer.pos++ // consume '!' + // Check if next token is an identifier (doesn't verify it's literally "important") let next_type = lexer.next_token_fast(true) // skip whitespace if (next_type === TOKEN_IDENT) { has_important = true @@ -208,11 +240,24 @@ export class DeclarationParser { lexer.next_token_fast(true) // Advance to next token after "important", skip whitespace break } + // '!' wasn't followed by an identifier - treat the already-peeked token as + // ordinary content; lexer.pos is already past it, so the raw scan picks up + // naturally (an extra next_token_fast here would swallow e.g. a trailing '}'). + last_end = lexer.token_end + value_end = last_end + continue } - last_end = lexer.token_end - value_end = last_end - lexer.next_token_fast(true) // skip whitespace + if (stop_ch === 0) { + // Ran out of input before finding any of the above + last_end = skip_whitespace_and_comments_backward(this.source, lexer.pos, value_start) + value_end = last_end + lexer.next_token_fast(false) // tokenize EOF so token_type reflects it + break + } + + // stop_ch is ';' or '}' but paren_depth !== 0 - ordinary content inside parens + lexer.pos++ } // Store value position (trimmed) and parse value nodes diff --git a/src/parse.ts b/src/parse.ts index aea38fc..d29627c 100644 --- a/src/parse.ts +++ b/src/parse.ts @@ -34,13 +34,14 @@ import { TOKEN_COLON, TOKEN_FUNCTION, } from './token-types' -import { trim_boundaries } from './parse-utils' +import { trim_boundaries, skip_whitespace_and_comments_backward } from './parse-utils' import { CHAR_PERIOD, CHAR_GREATER_THAN, CHAR_PLUS, CHAR_TILDE, CHAR_AMPERSAND, + CHAR_LEFT_BRACE, } from './string-utils' export interface ParserOptions { @@ -304,10 +305,13 @@ export class Parser { let selector_line = this.lexer.token_line let selector_column = this.lexer.token_column - // Consume tokens until we hit '{' let last_end = this.lexer.token_end - while (!this.is_eof() && this.peek_type() !== TOKEN_LEFT_BRACE) { - last_end = this.lexer.token_end + if (this.peek_type() !== TOKEN_LEFT_BRACE) { + // Fast-forward to the unquoted '{' (or EOF) — SelectorParser below re-tokenizes + // this span properly, so this coarse pass only needs the boundary. + this.lexer.skip_to_unquoted(CHAR_LEFT_BRACE) + last_end = skip_whitespace_and_comments_backward(this.source, this.lexer.pos, selector_start) + // Tokenize it so peek_type()/token_* reflect it, as the old loop would leave behind. this.next_token() } diff --git a/src/string-utils.ts b/src/string-utils.ts index 506c03d..d6b04f4 100644 --- a/src/string-utils.ts +++ b/src/string-utils.ts @@ -22,6 +22,12 @@ export const CHAR_DOLLAR = 0x24 // $ export const CHAR_CARET = 0x5e // ^ export const CHAR_COLON = 0x3a // : export const CHAR_LESS_THAN = 0x3c // < +export const CHAR_LEFT_BRACE = 0x7b // { +export const CHAR_SEMICOLON = 0x3b // ; +export const CHAR_RIGHT_BRACE = 0x7d // } +export const CHAR_LEFT_PAREN = 0x28 // ( +export const CHAR_RIGHT_PAREN = 0x29 // ) +export const CHAR_EXCLAMATION = 0x21 // ! /** * Check if a character code is whitespace (space, tab, newline, CR, or FF) diff --git a/src/tokenize.ts b/src/tokenize.ts index 5b24579..96b1a6f 100644 --- a/src/tokenize.ts +++ b/src/tokenize.ts @@ -74,6 +74,28 @@ const CHAR_CARRIAGE_RETURN = 0x0d // \r const CHAR_LINE_FEED = 0x0a // \n const CHAR_FORM_FEED = 0x0c // \f +// Characters skip_to_unquoted()/skip_to_declaration_stop() must stop and inspect: quotes, +// comment-starting '/', and newlines. One shared table keeps the ordinary-char fast path +// to a single lookup, same cost as consume_ident_or_function's inner loop. +const SKIP_SCAN_INTERESTING = new Uint8Array(128) +SKIP_SCAN_INTERESTING[CHAR_DOUBLE_QUOTE] = 1 +SKIP_SCAN_INTERESTING[CHAR_SINGLE_QUOTE] = 1 +SKIP_SCAN_INTERESTING[CHAR_FORWARD_SLASH] = 1 +SKIP_SCAN_INTERESTING[CHAR_LINE_FEED] = 1 +SKIP_SCAN_INTERESTING[CHAR_CARRIAGE_RETURN] = 1 +SKIP_SCAN_INTERESTING[CHAR_FORM_FEED] = 1 + +// Stop characters skip_to_declaration_stop() reports back: statement/block/paren +// delimiters plus '!' for !important. The caller (parse-declaration.ts) decides what +// each one means; this table only flags which are worth stopping for. +const DECL_VALUE_STOP = new Uint8Array(128) +DECL_VALUE_STOP[CHAR_SEMICOLON] = 1 +DECL_VALUE_STOP[CHAR_LEFT_BRACE] = 1 +DECL_VALUE_STOP[CHAR_RIGHT_BRACE] = 1 +DECL_VALUE_STOP[CHAR_LEFT_PAREN] = 1 +DECL_VALUE_STOP[CHAR_RIGHT_PAREN] = 1 +DECL_VALUE_STOP[CHAR_EXCLAMATION] = 1 + export interface LexerPosition { pos: number line: number @@ -143,24 +165,31 @@ export class Lexer { // Outer loop replaces comment recursion: after consuming a comment, continue // to find the actual next token instead of making a recursive call. while (true) { - // Inline whitespace skip — avoids method call + duplicate charCodeAt per char + // Inline whitespace skip, hoisted to locals (synced back once) instead of touching + // this.pos/_line/_line_offset per char — most runs are plain spaces. if (skip_whitespace) { - while (this.pos < source_length) { - let ch = source.charCodeAt(this.pos) + let pos = this.pos + let line = this._line + let line_offset = this._line_offset + while (pos < source_length) { + let ch = source.charCodeAt(pos) if (ch >= 128 || (char_types[ch] & (CHAR_WHITESPACE | CHAR_NEWLINE)) === 0) break - this.pos++ + pos++ if ((char_types[ch] & CHAR_NEWLINE) !== 0) { if ( ch === CHAR_CARRIAGE_RETURN && - this.pos < source_length && - source.charCodeAt(this.pos) === CHAR_LINE_FEED + pos < source_length && + source.charCodeAt(pos) === CHAR_LINE_FEED ) { - this.pos++ + pos++ } - this._line++ - this._line_offset = this.pos + line++ + line_offset = pos } } + this.pos = pos + this._line = line + this._line_offset = line_offset } if (this.pos >= source_length) { @@ -215,100 +244,47 @@ export class Lexer { this.pos + 1 < source_length && source.charCodeAt(this.pos + 1) === CHAR_ASTERISK ) { - let comment_start = start - let comment_line = start_line - let comment_column = start_column - - // Neither / nor * are newlines — safe to skip without newline tracking - this.pos += 2 - - // Use native string search to find */ — dramatically faster than a JS loop - // for typical comment bodies (SIMD-accelerated in V8). - let end_idx = source.indexOf('*/', this.pos) - if (end_idx < 0) { - // Unterminated comment: scan tail for newlines, advance to EOF - this._scan_newlines(this.pos, source_length) - this.pos = source_length - } else { - this._scan_newlines(this.pos, end_idx) - this.pos = end_idx + 2 - } - - if (this.on_comment) { - this.on_comment({ - start: comment_start, - end: this.pos, - length: this.pos - comment_start, - line: comment_line, - column: comment_column, - }) - } - + this._skip_comment() // Loop instead of recursing — eliminates stack frame overhead continue } - // Strings: " or ' - if (ch === CHAR_DOUBLE_QUOTE || ch === CHAR_SINGLE_QUOTE) { - return this.consume_string(ch, start_line, start_column) - } - - // Numbers: digit or . followed by digit - if (ch < 128 && (char_types[ch] & CHAR_DIGIT) !== 0) { - return this.consume_number(start_line, start_column) + // Identifier or function — checked first: the single most common non-whitespace + // token (~15-20% of all tokens), ahead of numbers/hashes/at-keywords/strings and + // the now-extinct CDO/CDC tokens that used to precede it (profiled on bootstrap.css/tailwind.css). + if (is_ident_start(ch)) { + return this.consume_ident_or_function(start_line, start_column) } - if (ch === CHAR_DOT) { + // Hyphen-led tokens: --custom-property (identifier), -5px (signed number), or the + // legacy --> CDC token — consolidated into one block sharing a single lookahead read. + // CDC must be checked first: "-->" would otherwise read as identifier "--" plus ">". + if (ch === CHAR_HYPHEN) { let next = this.pos + 1 < source_length ? source.charCodeAt(this.pos + 1) : 0 - if (next < 128 && (char_types[next] & CHAR_DIGIT) !== 0) { - return this.consume_number(start_line, start_column) - } - } - // CDO: - if (ch === CHAR_HYPHEN && this.pos + 2 < source_length) { - if ( - source.charCodeAt(this.pos + 1) === CHAR_HYPHEN && + next === CHAR_HYPHEN && + this.pos + 2 < source_length && source.charCodeAt(this.pos + 2) === CHAR_GREATER_THAN ) { - // --> contains no newlines + // CDC: --> (contains no newlines) this.pos += 3 return this.make_token(TOKEN_CDC, start, this.pos, start_line, start_column) } - } - - // At-keyword: @media, @keyframes, etc - if (ch === CHAR_AT_SIGN) { - return this.consume_at_keyword(start_line, start_column) - } - // Hash: #id or #fff - if (ch === CHAR_HASH) { - return this.consume_hash(start_line, start_column) - } - - // Identifier or function - if (is_ident_start(ch)) { - return this.consume_ident_or_function(start_line, start_column) - } - if (ch === CHAR_HYPHEN) { - let next = this.pos + 1 < source_length ? source.charCodeAt(this.pos + 1) : 0 if (is_ident_start(next) || next === CHAR_HYPHEN) { return this.consume_ident_or_function(start_line, start_column) } + + if (next < 128 && (char_types[next] & CHAR_DIGIT) !== 0) { + return this.consume_number(start_line, start_column) + } + if (next === CHAR_DOT) { + let next2 = this.pos + 2 < source_length ? source.charCodeAt(this.pos + 2) : 0 + if (next2 < 128 && (char_types[next2] & CHAR_DIGIT) !== 0) { + return this.consume_number(start_line, start_column) + } + } } // Backslash: escape sequence starting an identifier @@ -319,11 +295,22 @@ export class Lexer { } } - // Hyphen/Plus: could be signed number like -5 or +5 - if (ch === CHAR_HYPHEN || ch === CHAR_PLUS) { + // Numbers: digit or . followed by digit + if (ch < 128 && (char_types[ch] & CHAR_DIGIT) !== 0) { + return this.consume_number(start_line, start_column) + } + + if (ch === CHAR_DOT) { let next = this.pos + 1 < source_length ? source.charCodeAt(this.pos + 1) : 0 - let is_next_digit = next < 128 && (char_types[next] & CHAR_DIGIT) !== 0 - if (is_next_digit) { + if (next < 128 && (char_types[next] & CHAR_DIGIT) !== 0) { + return this.consume_number(start_line, start_column) + } + } + + // Plus: could be a signed number like +5 (the hyphen-led case is handled above) + if (ch === CHAR_PLUS) { + let next = this.pos + 1 < source_length ? source.charCodeAt(this.pos + 1) : 0 + if (next < 128 && (char_types[next] & CHAR_DIGIT) !== 0) { return this.consume_number(start_line, start_column) } if (next === CHAR_DOT) { @@ -334,6 +321,34 @@ export class Lexer { } } + // Strings: " or ' + if (ch === CHAR_DOUBLE_QUOTE || ch === CHAR_SINGLE_QUOTE) { + return this.consume_string(ch, start_line, start_column) + } + + // At-keyword: @media, @keyframes, etc + if (ch === CHAR_AT_SIGN) { + return this.consume_at_keyword(start_line, start_column) + } + + // Hash: #id or #fff + if (ch === CHAR_HASH) { + return this.consume_hash(start_line, start_column) + } + + // CDO: