From 60e4bffd74dcfa850327341c4538ae016274ed8b Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Wed, 29 Jul 2026 09:39:10 +0000
Subject: [PATCH 1/4] Reorder tokenizer dispatch chain by measured token
 frequency

Profiled parsing bootstrap.css/tailwind.css: next_token_fast (21.3%)
and consume_ident_or_function (20.1%) dominate runtime. Token-frequency
analysis on the same files showed identifiers are the single most
common non-whitespace token (~15-21%), yet the dispatch chain checked
them 11th - behind CDO/CDC (`<!--`/`-->`, which never occurred in
either file - a legacy token from `<style><!--...--></style>`
compatibility) and the at-keyword/hash checks, both individually rarer
than identifiers.

Reordered the chain to check identifiers right after the
comment/whitespace fast paths, and moved CDO to the very end, right
before the delimiter fallback. Consolidated the three separate
`ch === CHAR_HYPHEN` checks (CDC, hyphen-led identifier, hyphen-led
signed number) into one block sharing a single lookahead read, instead
of three independent re-reads - while preserving the exact original
precedence (CDC must still win for "-->", since a bare "--" is valid
identifier-start for custom properties like `--custom-color`).

Two other approaches were measured and rejected before landing on this:
- Sticky-regex-based identifier scanning was 35-40% *slower* than the
  existing manual char-loop, even accounting for real identifiers
  averaging ~11-12 chars (longer than assumed) - V8's JIT-compiled
  loop with a lookup table already beats regex .exec() overhead here.
- Pre-scanning the source once for "<!--"/"-->" and gating the CDO
  check behind a boolean flag measured slightly *slower* than the bare
  comparison - branch prediction already makes an always-false check
  near-free, so the extra flag read is pure overhead.

Net effect verified via 6 repeated full end-to-end parse benchmarks
(3 before, 3 after) on both files together: ~1.5-4% faster depending
on file, with bootstrap.css seeing the larger gain (tailwind.css's
much higher rate of backslash-escaped identifiers, from Tailwind's
`\:` variant syntax, doesn't benefit from the identifier-check reorder
since escaped idents take a separate path). Full test suite (1386
tests, including dedicated CDO/CDC and custom-property tokenizer
tests) passes unchanged.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017XjfDK7or6VrnJ41vBhxrK
---
 src/tokenize.ts | 122 ++++++++++++++++++++++++++++--------------------
 1 file changed, 71 insertions(+), 51 deletions(-)

diff --git a/src/tokenize.ts b/src/tokenize.ts
index 5b24579..2967572 100644
--- a/src/tokenize.ts
+++ b/src/tokenize.ts
@@ -248,67 +248,47 @@ export class Lexer {
 				continue
 			}
 
-			// Strings: " or '
-			if (ch === CHAR_DOUBLE_QUOTE || ch === CHAR_SINGLE_QUOTE) {
-				return this.consume_string(ch, start_line, start_column)
-			}
-
-			// Numbers: digit or . followed by digit
-			if (ch < 128 && (char_types[ch] & CHAR_DIGIT) !== 0) {
-				return this.consume_number(start_line, start_column)
+			// Identifier or function — checked early: across real-world CSS this is the single
+			// most common non-whitespace token (~15-20%+ of all tokens), well ahead of numbers,
+			// hashes, at-keywords, strings, and the (essentially extinct in modern CSS) CDO/CDC
+			// tokens that used to precede it here. Measured via profiling + token frequency
+			// analysis on bootstrap.css/tailwind.css — see PR discussion for numbers.
+			if (is_ident_start(ch)) {
+				return this.consume_ident_or_function(start_line, start_column)
 			}
 
-			if (ch === CHAR_DOT) {
+			// Hyphen-led tokens: --custom-property/-webkit-prefix (identifier), -5px (signed
+			// number), or the legacy --> CDC token. Consolidated into one block (single charCodeAt
+			// for the lookahead char) instead of three separate `ch === CHAR_HYPHEN` checks.
+			// NOTE: CDC must be checked first — "-->" would otherwise be misread as identifier "--"
+			// followed by a stray ">" delimiter, since -- is itself valid identifier-start (custom
+			// properties).
+			if (ch === CHAR_HYPHEN) {
 				let next = this.pos + 1 < source_length ? source.charCodeAt(this.pos + 1) : 0
-				if (next < 128 && (char_types[next] & CHAR_DIGIT) !== 0) {
-					return this.consume_number(start_line, start_column)
-				}
-			}
-
-			// CDO: <!--
-			if (ch === CHAR_LESS_THAN && this.pos + 3 < source_length) {
-				if (
-					source.charCodeAt(this.pos + 1) === CHAR_EXCLAMATION &&
-					source.charCodeAt(this.pos + 2) === CHAR_HYPHEN &&
-					source.charCodeAt(this.pos + 3) === CHAR_HYPHEN
-				) {
-					// <!-- contains no newlines
-					this.pos += 4
-					return this.make_token(TOKEN_CDO, start, this.pos, start_line, start_column)
-				}
-			}
 
-			// CDC: -->
-			if (ch === CHAR_HYPHEN && this.pos + 2 < source_length) {
 				if (
-					source.charCodeAt(this.pos + 1) === CHAR_HYPHEN &&
+					next === CHAR_HYPHEN &&
+					this.pos + 2 < source_length &&
 					source.charCodeAt(this.pos + 2) === CHAR_GREATER_THAN
 				) {
-					// --> contains no newlines
+					// CDC: --> (contains no newlines)
 					this.pos += 3
 					return this.make_token(TOKEN_CDC, start, this.pos, start_line, start_column)
 				}
-			}
-
-			// At-keyword: @media, @keyframes, etc
-			if (ch === CHAR_AT_SIGN) {
-				return this.consume_at_keyword(start_line, start_column)
-			}
-
-			// Hash: #id or #fff
-			if (ch === CHAR_HASH) {
-				return this.consume_hash(start_line, start_column)
-			}
 
-			// Identifier or function
-			if (is_ident_start(ch)) {
-				return this.consume_ident_or_function(start_line, start_column)
-			}
-			if (ch === CHAR_HYPHEN) {
-				let next = this.pos + 1 < source_length ? source.charCodeAt(this.pos + 1) : 0
 				if (is_ident_start(next) || next === CHAR_HYPHEN) {
 					return this.consume_ident_or_function(start_line, start_column)
 				}
+
+				if (next < 128 && (char_types[next] & CHAR_DIGIT) !== 0) {
+					return this.consume_number(start_line, start_column)
+				}
+				if (next === CHAR_DOT) {
+					let next2 = this.pos + 2 < source_length ? source.charCodeAt(this.pos + 2) : 0
+					if (next2 < 128 && (char_types[next2] & CHAR_DIGIT) !== 0) {
+						return this.consume_number(start_line, start_column)
+					}
+				}
 			}
 
 			// Backslash: escape sequence starting an identifier
@@ -319,11 +299,22 @@ export class Lexer {
 				}
 			}
 
-			// Hyphen/Plus: could be signed number like -5 or +5
-			if (ch === CHAR_HYPHEN || ch === CHAR_PLUS) {
+			// Numbers: digit or . followed by digit
+			if (ch < 128 && (char_types[ch] & CHAR_DIGIT) !== 0) {
+				return this.consume_number(start_line, start_column)
+			}
+
+			if (ch === CHAR_DOT) {
+				let next = this.pos + 1 < source_length ? source.charCodeAt(this.pos + 1) : 0
+				if (next < 128 && (char_types[next] & CHAR_DIGIT) !== 0) {
+					return this.consume_number(start_line, start_column)
+				}
+			}
+
+			// Plus: could be a signed number like +5 (the hyphen-led case is handled above)
+			if (ch === CHAR_PLUS) {
 				let next = this.pos + 1 < source_length ? source.charCodeAt(this.pos + 1) : 0
-				let is_next_digit = next < 128 && (char_types[next] & CHAR_DIGIT) !== 0
-				if (is_next_digit) {
+				if (next < 128 && (char_types[next] & CHAR_DIGIT) !== 0) {
 					return this.consume_number(start_line, start_column)
 				}
 				if (next === CHAR_DOT) {
@@ -334,6 +325,35 @@ export class Lexer {
 				}
 			}
 
+			// Strings: " or '
+			if (ch === CHAR_DOUBLE_QUOTE || ch === CHAR_SINGLE_QUOTE) {
+				return this.consume_string(ch, start_line, start_column)
+			}
+
+			// At-keyword: @media, @keyframes, etc
+			if (ch === CHAR_AT_SIGN) {
+				return this.consume_at_keyword(start_line, start_column)
+			}
+
+			// Hash: #id or #fff
+			if (ch === CHAR_HASH) {
+				return this.consume_hash(start_line, start_column)
+			}
+
+			// CDO: <!-- (essentially extinct in modern CSS — checked last, right before the
+			// default delimiter fallback; a lone "<" that isn't CDO just falls through to DELIM)
+			if (ch === CHAR_LESS_THAN && this.pos + 3 < source_length) {
+				if (
+					source.charCodeAt(this.pos + 1) === CHAR_EXCLAMATION &&
+					source.charCodeAt(this.pos + 2) === CHAR_HYPHEN &&
+					source.charCodeAt(this.pos + 3) === CHAR_HYPHEN
+				) {
+					// <!-- contains no newlines
+					this.pos += 4
+					return this.make_token(TOKEN_CDO, start, this.pos, start_line, start_column)
+				}
+			}
+
 			// Default: delimiter
 			this.pos++
 			return this.make_token(TOKEN_DELIM, start, this.pos, start_line, start_column)

From 865fbd6e199e4aab473069cd6ea4d53360ca234f Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Wed, 29 Jul 2026 09:49:30 +0000
Subject: [PATCH 2/4] Hoist hot-loop position/line tracking to local variables
 in the tokenizer

Continuing the tokenizer tuning pass: measured whether reading/writing
this.pos (and this._line/this._line_offset for loops that track
newlines) on every character, versus a local variable synced back once
at the end, makes a difference in V8. It does - roughly 5% faster in
isolation for a full linear identifier-class scan, consistent between
bootstrap.css and tailwind.css.

Applied to every consume_* method that showed measurable self-time in
a CPU profile of parsing both files: consume_ident_or_function (20.1%
self-time), consume_number (4.7%), the inline whitespace-skip in
next_token_fast, and consume_whitespace. Also did the same (mechanical,
same-cost-to-verify) for consume_hash and consume_at_keyword despite
their low frequency in real CSS, since the pattern is safe and free.

consume_string, consume_unicode_range, and consume_hex_escape were
deliberately left untouched: none showed measurable self-time in the
profiler even before this change (well under 0.4% combined), and their
escape-sequence/newline interactions are more tangled, so there's
nothing to gain versus real risk to a hand-tuned hot path.

For functions with no newline tracking in their hot path (consume_number,
consume_at_keyword, consume_hash), the local variable is used
throughout with a single sync-back before each return. For
consume_ident_or_function, the common (non-escape) branch uses the
local var; the rare backslash-escape branch syncs this.pos before
using the existing this.pos/this.advance()-based logic (which already
handles newline tracking correctly), then resyncs the local afterward.
For the whitespace-skip loops, pos/_line/_line_offset are all hoisted
together and synced back once after the loop.

Verified via 6 repeated full end-to-end parse benchmarks (bootstrap.css
+ tailwind.css together) before and after: combined with the prior
dispatch-reorder commit, total measured improvement is ~5.9% (versus
~1.5-4% from the reorder alone). Full test suite (1386 tests) passes
unchanged.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017XjfDK7or6VrnJ41vBhxrK
---
 src/tokenize.ts | 161 ++++++++++++++++++++++++++++--------------------
 1 file changed, 94 insertions(+), 67 deletions(-)

diff --git a/src/tokenize.ts b/src/tokenize.ts
index 2967572..f61fd1c 100644
--- a/src/tokenize.ts
+++ b/src/tokenize.ts
@@ -143,24 +143,32 @@ export class Lexer {
 		// Outer loop replaces comment recursion: after consuming a comment, continue
 		// to find the actual next token instead of making a recursive call.
 		while (true) {
-			// Inline whitespace skip — avoids method call + duplicate charCodeAt per char
+			// Inline whitespace skip — avoids method call + duplicate charCodeAt per char.
+			// Hoisted to locals (synced back once) instead of touching this.pos/_line/_line_offset
+			// on every character — most runs are plain spaces that never hit the newline branch.
 			if (skip_whitespace) {
-				while (this.pos < source_length) {
-					let ch = source.charCodeAt(this.pos)
+				let pos = this.pos
+				let line = this._line
+				let line_offset = this._line_offset
+				while (pos < source_length) {
+					let ch = source.charCodeAt(pos)
 					if (ch >= 128 || (char_types[ch] & (CHAR_WHITESPACE | CHAR_NEWLINE)) === 0) break
-					this.pos++
+					pos++
 					if ((char_types[ch] & CHAR_NEWLINE) !== 0) {
 						if (
 							ch === CHAR_CARRIAGE_RETURN &&
-							this.pos < source_length &&
-							source.charCodeAt(this.pos) === CHAR_LINE_FEED
+							pos < source_length &&
+							source.charCodeAt(pos) === CHAR_LINE_FEED
 						) {
-							this.pos++
+							pos++
 						}
-						this._line++
-						this._line_offset = this.pos
+						line++
+						line_offset = pos
 					}
 				}
+				this.pos = pos
+				this._line = line
+				this._line_offset = line_offset
 			}
 
 			if (this.pos >= source_length) {
@@ -386,24 +394,31 @@ export class Lexer {
 		const source = this.source
 		const source_length = source.length
 		let start = this.pos
+		// Hoisted to locals (synced back once), same as next_token_fast's inline skip.
+		let pos = start
+		let line = this._line
+		let line_offset = this._line_offset
 		// Inline advance: read ch once, pos++, then track newlines with already-read value
-		while (this.pos < source_length) {
-			let ch = source.charCodeAt(this.pos)
+		while (pos < source_length) {
+			let ch = source.charCodeAt(pos)
 			if (ch >= 128 || (char_types[ch] & (CHAR_WHITESPACE | CHAR_NEWLINE)) === 0) break
-			this.pos++
+			pos++
 			if ((char_types[ch] & CHAR_NEWLINE) !== 0) {
 				if (
 					ch === CHAR_CARRIAGE_RETURN &&
-					this.pos < source_length &&
-					source.charCodeAt(this.pos) === CHAR_LINE_FEED
+					pos < source_length &&
+					source.charCodeAt(pos) === CHAR_LINE_FEED
 				) {
-					this.pos++
+					pos++
 				}
-				this._line++
-				this._line_offset = this.pos
+				line++
+				line_offset = pos
 			}
 		}
-		return this.make_token(TOKEN_WHITESPACE, start, this.pos, start_line, start_column)
+		this.pos = pos
+		this._line = line
+		this._line_offset = line_offset
+		return this.make_token(TOKEN_WHITESPACE, start, pos, start_line, start_column)
 	}
 
 	consume_string(quote: number, start_line: number, start_column: number): TokenType {
@@ -496,98 +511,106 @@ export class Lexer {
 		const source = this.source
 		const source_length = source.length
 		let start = this.pos
+		// Numbers never contain newlines, so the whole scan can run on a local var —
+		// no line/column bookkeeping to keep in sync with this.pos along the way.
+		let pos = start
 
 		// Optional sign — + and - are never newlines
-		let ch = source.charCodeAt(this.pos)
+		let ch = source.charCodeAt(pos)
 		if (ch === CHAR_PLUS || ch === CHAR_HYPHEN) {
-			this.pos++
+			pos++
 		}
 
 		// Integer part — digits are never newlines, use pos++
-		while (this.pos < source_length) {
-			let ch = source.charCodeAt(this.pos)
+		while (pos < source_length) {
+			let ch = source.charCodeAt(pos)
 			if (ch >= 128 || (char_types[ch] & CHAR_DIGIT) === 0) break
-			this.pos++
+			pos++
 		}
 
 		// Decimal part
-		if (
-			this.pos < source_length &&
-			source.charCodeAt(this.pos) === CHAR_DOT &&
-			this.pos + 1 < source_length
-		) {
-			let next = source.charCodeAt(this.pos + 1)
+		if (pos < source_length && source.charCodeAt(pos) === CHAR_DOT && pos + 1 < source_length) {
+			let next = source.charCodeAt(pos + 1)
 			if (next < 128 && (char_types[next] & CHAR_DIGIT) !== 0) {
-				this.pos++ // . is never a newline
-				while (this.pos < source_length) {
-					let ch = source.charCodeAt(this.pos)
+				pos++ // . is never a newline
+				while (pos < source_length) {
+					let ch = source.charCodeAt(pos)
 					if (ch >= 128 || (char_types[ch] & CHAR_DIGIT) === 0) break
-					this.pos++ // digits: never newlines
+					pos++ // digits: never newlines
 				}
 			}
 		}
 
 		// Exponent: e or E
-		if (this.pos < source_length) {
-			let ch = source.charCodeAt(this.pos)
+		if (pos < source_length) {
+			let ch = source.charCodeAt(pos)
 			if (ch === CHAR_LOWERCASE_E || ch === CHAR_UPPERCASE_E) {
-				let next = this.pos + 1 < source_length ? source.charCodeAt(this.pos + 1) : 0
+				let next = pos + 1 < source_length ? source.charCodeAt(pos + 1) : 0
 				let is_next_digit = next < 128 && (char_types[next] & CHAR_DIGIT) !== 0
-				let next2 = this.pos + 2 < source_length ? source.charCodeAt(this.pos + 2) : 0
+				let next2 = pos + 2 < source_length ? source.charCodeAt(pos + 2) : 0
 				let is_next2_digit = next2 < 128 && (char_types[next2] & CHAR_DIGIT) !== 0
 				if (is_next_digit || ((next === CHAR_PLUS || next === CHAR_HYPHEN) && is_next2_digit)) {
-					this.pos++ // e/E — never a newline
-					if (this.pos < source_length) {
-						let sign = source.charCodeAt(this.pos)
+					pos++ // e/E — never a newline
+					if (pos < source_length) {
+						let sign = source.charCodeAt(pos)
 						if (sign === CHAR_PLUS || sign === CHAR_HYPHEN) {
-							this.pos++ // +/- — never a newline
+							pos++ // +/- — never a newline
 						}
 					}
-					while (this.pos < source_length) {
-						let ch = source.charCodeAt(this.pos)
+					while (pos < source_length) {
+						let ch = source.charCodeAt(pos)
 						if (ch >= 128 || (char_types[ch] & CHAR_DIGIT) === 0) break
-						this.pos++ // digits: never newlines
+						pos++ // digits: never newlines
 					}
 				}
 			}
 		}
 
 		// Check for unit (dimension) or percentage
-		if (this.pos < source_length) {
-			let ch = source.charCodeAt(this.pos)
+		if (pos < source_length) {
+			let ch = source.charCodeAt(pos)
 			if (ch === CHAR_PERCENT) {
-				this.pos++ // % is never a newline
-				return this.make_token(TOKEN_PERCENTAGE, start, this.pos, start_line, start_column)
+				pos++ // % is never a newline
+				this.pos = pos
+				return this.make_token(TOKEN_PERCENTAGE, start, pos, start_line, start_column)
 			}
 			if (
 				is_ident_start(ch) ||
 				(ch === CHAR_HYPHEN &&
-					is_ident_start(this.pos + 1 < source_length ? source.charCodeAt(this.pos + 1) : 0))
+					is_ident_start(pos + 1 < source_length ? source.charCodeAt(pos + 1) : 0))
 			) {
 				// Unit: px, em, rem, etc — ident chars and non-ASCII are never newlines
-				while (this.pos < source_length) {
-					let ch = source.charCodeAt(this.pos)
+				while (pos < source_length) {
+					let ch = source.charCodeAt(pos)
 					if (ch < 0x80 && (char_types[ch] & CHAR_IDENT) === 0) break
-					this.pos++
+					pos++
 				}
-				return this.make_token(TOKEN_DIMENSION, start, this.pos, start_line, start_column)
+				this.pos = pos
+				return this.make_token(TOKEN_DIMENSION, start, pos, start_line, start_column)
 			}
 		}
 
-		return this.make_token(TOKEN_NUMBER, start, this.pos, start_line, start_column)
+		this.pos = pos
+		return this.make_token(TOKEN_NUMBER, start, pos, start_line, start_column)
 	}
 
 	consume_ident_or_function(start_line: number, start_column: number): TokenType {
 		const source = this.source
 		const source_length = source.length
 		let start = this.pos
+		// Hoisted to a local: the common case below (plain ident chars, no escapes) never
+		// touches line/column, so it can advance a local var instead of this.pos on every
+		// char, syncing back only when the rare backslash-escape path needs this.pos (which
+		// it reads/writes directly, including via this.advance() for line tracking).
+		let pos = start
 
 		// Consume identifier (with escape sequence support)
-		while (this.pos < source_length) {
-			let ch = source.charCodeAt(this.pos)
+		while (pos < source_length) {
+			let ch = source.charCodeAt(pos)
 
 			// Handle escape sequences: \ followed by hex digits or any character
 			if (ch === CHAR_BACKSLASH) {
+				this.pos = pos
 				if (this.pos + 1 >= source_length) break
 
 				let next = source.charCodeAt(this.pos + 1)
@@ -615,14 +638,16 @@ export class Lexer {
 					// Non-newline chars: safe to pos++
 					this.pos++
 				}
+				pos = this.pos
 			} else if (ch >= 0x80 || (char_types[ch] & CHAR_IDENT) !== 0) {
 				// Normal identifier character — ident chars (a-z,A-Z,0-9,-,_) and
 				// non-ASCII code units are never newlines (newlines are 0x0A/0x0D/0x0C)
-				this.pos++
+				pos++
 			} else {
 				break
 			}
 		}
+		this.pos = pos
 
 		// Check for unicode-range: u+ or U+
 		if (this.pos - start === 1) {
@@ -700,32 +725,34 @@ export class Lexer {
 		const source = this.source
 		const source_length = source.length
 		let start = this.pos
-		this.pos++ // Skip @ — never a newline
+		let pos = start + 1 // Skip @ — never a newline
 
 		// Ident chars (a-z,A-Z,0-9,-,_) and non-ASCII are never newlines — use pos++
-		while (this.pos < source_length) {
-			let ch = source.charCodeAt(this.pos)
+		while (pos < source_length) {
+			let ch = source.charCodeAt(pos)
 			if (ch < 0x80 && (char_types[ch] & CHAR_IDENT) === 0) break
-			this.pos++
+			pos++
 		}
 
-		return this.make_token(TOKEN_AT_KEYWORD, start, this.pos, start_line, start_column)
+		this.pos = pos
+		return this.make_token(TOKEN_AT_KEYWORD, start, pos, start_line, start_column)
 	}
 
 	consume_hash(start_line: number, start_column: number): TokenType {
 		const source = this.source
 		const source_length = source.length
 		let start = this.pos
-		this.pos++ // Skip # — never a newline
+		let pos = start + 1 // Skip # — never a newline
 
 		// Ident chars and non-ASCII are never newlines — use pos++
-		while (this.pos < source_length) {
-			let ch = source.charCodeAt(this.pos)
+		while (pos < source_length) {
+			let ch = source.charCodeAt(pos)
 			if (ch < 0x80 && (char_types[ch] & CHAR_IDENT) === 0) break
-			this.pos++
+			pos++
 		}
 
-		return this.make_token(TOKEN_HASH, start, this.pos, start_line, start_column)
+		this.pos = pos
+		return this.make_token(TOKEN_HASH, start, pos, start_line, start_column)
 	}
 
 	advance(count: number = 1): void {

From 44bf094c59a748f4a86c7d2ce854fa5a8de67148 Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Wed, 29 Jul 2026 10:40:29 +0000
Subject: [PATCH 3/4] Eliminate double-tokenization at selector and
 declaration-value boundaries

The parser previously did a coarse boundary scan by fully tokenizing
(next_token/next_token_fast) up to a selector's `{` or a declaration
value's terminator, only to hand that exact span to a dedicated
sub-parser (SelectorParser, ValueNodeParser) that re-tokenizes it from
scratch to build the real AST. The coarse pass only ever needed to know
*where* the construct ends, not the individual tokens in it.

Replaced both boundary scans with raw character-level scans that skip
ordinary content via a single lookup-table check per character (same
cost profile as consume_ident_or_function's inner loop), while still
routing through consume_string/_skip_comment for correctness on quotes
and comments so escapes, nesting, and on_comment callbacks stay
byte-for-byte identical to the old tokenized scan:

- Lexer.skip_to_unquoted(target) - used by parse_selector to find the
  next unquoted '{'.
- Lexer.skip_to_declaration_stop(end) - used by parse_declaration_with_lexer
  to find the next unquoted ';', '{', '}', '(', ')', or '!', with paren-depth
  tracking left to the caller (needed for e.g. url(data:...;...)).
- _skip_comment() extracted from next_token_fast's inline comment handling
  so both the tokenizer and the new raw scans share one implementation.

Getting this right required matching several non-obvious side effects of
the old token-based scan exactly (verified by diffing serialized ASTs
between old and new implementations across handwritten edge cases and
the full text of bootstrap/tailwind's regular and minified builds - byte
identical in both):
- The old "consume ':', skip whitespace" call also tokenized the value's
  first token as a side effect; the new scan needs to see that character
  itself (e.g. the '(' of a leading calc(...)), so whitespace is now
  skipped without tokenizing.
- A paren-depth adjustment can land exactly on the scan's `end` boundary
  without the loop's "ran out of input" branch ever running, so the loop
  now relies solely on explicit break/return/continue rather than a loop
  condition.
- "color:}" (empty value directly followed by the block's closing brace)
  is a quirk of the old pre-tokenization: the brace itself became the
  "first value token" before the main loop ever ran, extending the
  declaration's recorded text by one character. Replicated explicitly for
  that degenerate case.
- After '!' turns out not to be followed by an identifier, the
  already-peeked token must be treated as ordinary content in place,
  not consumed a second time - otherwise a trailing '}' right after
  a bare '!' gets swallowed instead of ending the declaration.

Measured on repeated bootstrap.css + tailwind.css parses: ~7.9% faster
than the tokenizer-only optimizations in the prior two commits (the
selector-boundary change alone measured ~3.83%; the declaration-value
scan accounts for most of the remainder).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017XjfDK7or6VrnJ41vBhxrK
---
 src/parse-declaration.ts | 130 ++++++++++++++++------
 src/parse.ts             |  14 ++-
 src/string-utils.ts      |   6 +
 src/tokenize.ts          | 229 ++++++++++++++++++++++++++++++++++-----
 4 files changed, 312 insertions(+), 67 deletions(-)

diff --git a/src/parse-declaration.ts b/src/parse-declaration.ts
index d08122b..72191ca 100644
--- a/src/parse-declaration.ts
+++ b/src/parse-declaration.ts
@@ -2,15 +2,20 @@
 import { Lexer } from './tokenize'
 import { CSSDataArena, DECLARATION, RAW, FLAG_IMPORTANT, FLAG_BROWSERHACK } from './arena'
 import { ValueParser } from './parse-value'
-import { is_vendor_prefixed } from './string-utils'
+import {
+	is_vendor_prefixed,
+	CHAR_LEFT_BRACE,
+	CHAR_RIGHT_BRACE,
+	CHAR_SEMICOLON,
+	CHAR_LEFT_PAREN,
+	CHAR_RIGHT_PAREN,
+	CHAR_EXCLAMATION,
+} from './string-utils'
 import {
 	TOKEN_IDENT,
 	TOKEN_COLON,
 	TOKEN_SEMICOLON,
 	TOKEN_DELIM,
-	TOKEN_EOF,
-	TOKEN_LEFT_BRACE,
-	TOKEN_RIGHT_BRACE,
 	TOKEN_LEFT_PAREN,
 	TOKEN_RIGHT_PAREN,
 	TOKEN_LEFT_BRACKET,
@@ -18,10 +23,9 @@ import {
 	TOKEN_COMMA,
 	TOKEN_HASH,
 	TOKEN_AT_KEYWORD,
-	TOKEN_FUNCTION,
 	type TokenType,
 } from './token-types'
-import { trim_boundaries } from './parse-utils'
+import { trim_boundaries, skip_whitespace_and_comments_backward } from './parse-utils'
 import { CSSNode } from './css-node'
 import type { Declaration } from './node-types'
 
@@ -145,7 +149,13 @@ export class DeclarationParser {
 			lexer.restore_position(has_delimiter_prefix ? initial_saved : saved)
 			return null
 		}
-		lexer.next_token_fast(true) // consume ':', skip whitespace
+		// Skip whitespace/comments after ':' WITHOUT tokenizing the first value token - the
+		// raw scan below (skip_to_declaration_stop) needs to see every character of the value
+		// uniformly, including what would otherwise become an already-consumed first token
+		// (e.g. a leading "calc(" - its '(' must reach the paren-depth tracking below, which
+		// it wouldn't if next_token_fast had already tokenized past it here).
+		lexer.pos = lexer.token_end // move past ':' (already at this position, but be explicit)
+		lexer.skip_whitespace_in_range(this.source.length)
 
 		// Create declaration node (length will be set later)
 		let declaration = this.arena.create_node(
@@ -160,47 +170,83 @@ export class DeclarationParser {
 		this.arena.set_content_start_delta(declaration, 0)
 		this.arena.set_content_length(declaration, prop_end - prop_start)
 
-		// Track value start (after colon, skipping whitespace)
-		// CRITICAL: Capture line/column for value parsing
-		// After consuming ':', lexer is now positioned at first value token
-		let value_start = lexer.token_start
-		let value_start_line = lexer.token_line
-		let value_start_column = lexer.token_column
+		// Track value start (after colon, skipping whitespace) - CRITICAL: Capture line/column
+		// for value parsing. Lexer is now positioned at the first value character (untokenized).
+		let value_start = lexer.pos
+		let value_start_line = lexer.line
+		let value_start_column = lexer.column
 		let value_end = value_start
 
 		// Parse value (everything until ';' or EOF)
 		let has_important = false
-		let last_end = lexer.token_end
+		let last_end = value_start
 		// Track parenthesis depth to handle semicolons inside functions (e.g., url(data:image/png;base64,...))
-		// NOTE: Same pattern exists in parse.ts for at-rule prelude parsing - keep in sync
 		let paren_depth = 0
 
-		// Process tokens until we hit semicolon, EOF, or end of input
-		while ((lexer.token_type as TokenType) !== TOKEN_EOF && lexer.token_start < end) {
-			let token_type = lexer.token_type as TokenType
-
-			// Track parenthesis depth
-			if (token_type === TOKEN_LEFT_PAREN || token_type === TOKEN_FUNCTION) {
+		// Fast-forward through the value using a raw character scan for the exact stop
+		// points the token-by-token loop used to check (paren depth, ';'/'}' at depth zero,
+		// an unparenthesized '{', and '!' for !important) - the ordinary content in between
+		// doesn't need full tokenization here, since ValueNodeParser re-tokenizes the
+		// resulting span properly afterward.
+		// NOTE: every exit is an explicit break/return - the "ran out of input" case is only
+		// detected via skip_to_declaration_stop returning 0, not via a loop condition, since a
+		// paren_depth adjustment can land exactly on `end` without going through that path.
+		while (true) {
+			let stop_ch = lexer.skip_to_declaration_stop(end)
+
+			if (stop_ch === CHAR_LEFT_PAREN) {
 				paren_depth++
-			} else if (token_type === TOKEN_RIGHT_PAREN) {
+				lexer.pos++
+				continue
+			}
+			if (stop_ch === CHAR_RIGHT_PAREN) {
 				paren_depth--
+				lexer.pos++
+				continue
 			}
 
-			// Only break on semicolon/brace when outside all parentheses
-			if (token_type === TOKEN_SEMICOLON && paren_depth === 0) break
-			if (token_type === TOKEN_RIGHT_BRACE && paren_depth === 0) break
+			if (stop_ch === CHAR_SEMICOLON && paren_depth === 0) {
+				value_end = skip_whitespace_and_comments_backward(this.source, lexer.pos, value_start)
+				// Tokenize ';' so token_type reflects it, matching the state the old
+				// token-by-token loop left behind right when its condition saw TOKEN_SEMICOLON
+				// (already tokenized, not yet consumed) - the unchanged code below relies on it.
+				lexer.next_token_fast(false)
+				break
+			}
 
-			// If we encounter '{', this is actually a style rule, not a declaration
-			if (token_type === TOKEN_LEFT_BRACE) {
+			if (stop_ch === CHAR_RIGHT_BRACE && paren_depth === 0) {
+				if (lexer.pos === value_start) {
+					// Degenerate case: colon directly followed by the block's closing brace
+					// with no value content at all (e.g. "color:}"). Replicates a quirk of the
+					// original token-based scan: the pre-tokenized "first value token" being
+					// '}' itself meant its end position became last_end before the per-token
+					// loop ever got a chance to run.
+					last_end = lexer.pos + 1
+					value_end = value_start
+				} else {
+					last_end = skip_whitespace_and_comments_backward(this.source, lexer.pos, value_start)
+					value_end = last_end
+				}
+				// Tokenize '}' so the caller's peek_type() sees it correctly - this declaration
+				// ends at a block boundary with no trailing semicolon, and '}' isn't consumed
+				// here (the enclosing block's own loop needs to see it to know it's done).
+				lexer.next_token_fast(false)
+				break
+			}
+
+			if (stop_ch === CHAR_LEFT_BRACE) {
+				// This is actually a style rule, not a declaration - '{' can appear at any
+				// paren depth and always bails out, regardless of depth.
 				lexer.restore_position(saved)
 				return null
 			}
 
-			// Check for ! followed by any identifier (optimized: only check when we see '!')
-			if (token_type === TOKEN_DELIM && this.source[lexer.token_start] === '!') {
+			if (stop_ch === CHAR_EXCLAMATION) {
 				// Mark end of value before !important
-				value_end = lexer.token_start
-				// Check if next token is an identifier
+				value_end = lexer.pos
+				lexer.pos++ // consume '!'
+				// Check if next token is an identifier (doesn't verify it's literally
+				// "important" - matches the pre-existing behavior this replaces)
 				let next_type = lexer.next_token_fast(true) // skip whitespace
 				if (next_type === TOKEN_IDENT) {
 					has_important = true
@@ -208,11 +254,27 @@ export class DeclarationParser {
 					lexer.next_token_fast(true) // Advance to next token after "important", skip whitespace
 					break
 				}
+				// '!' wasn't followed by an identifier - treat the already-peeked token as
+				// ordinary content. lexer.pos is already right after it (tokenizing to peek
+				// advances past it), so the raw scan below picks up from there naturally - an
+				// extra next_token_fast here would blindly consume whatever comes next (e.g. a
+				// block's closing '}') without the scan ever getting a chance to treat it as a
+				// stop condition.
+				last_end = lexer.token_end
+				value_end = last_end
+				continue
 			}
 
-			last_end = lexer.token_end
-			value_end = last_end
-			lexer.next_token_fast(true) // skip whitespace
+			if (stop_ch === 0) {
+				// Ran out of input before finding any of the above
+				last_end = skip_whitespace_and_comments_backward(this.source, lexer.pos, value_start)
+				value_end = last_end
+				lexer.next_token_fast(false) // tokenize EOF so token_type reflects it
+				break
+			}
+
+			// stop_ch is ';' or '}' but paren_depth !== 0 - ordinary content inside parens
+			lexer.pos++
 		}
 
 		// Store value position (trimmed) and parse value nodes
diff --git a/src/parse.ts b/src/parse.ts
index aea38fc..bbd5ec5 100644
--- a/src/parse.ts
+++ b/src/parse.ts
@@ -34,13 +34,14 @@ import {
 	TOKEN_COLON,
 	TOKEN_FUNCTION,
 } from './token-types'
-import { trim_boundaries } from './parse-utils'
+import { trim_boundaries, skip_whitespace_and_comments_backward } from './parse-utils'
 import {
 	CHAR_PERIOD,
 	CHAR_GREATER_THAN,
 	CHAR_PLUS,
 	CHAR_TILDE,
 	CHAR_AMPERSAND,
+	CHAR_LEFT_BRACE,
 } from './string-utils'
 
 export interface ParserOptions {
@@ -304,10 +305,15 @@ export class Parser {
 		let selector_line = this.lexer.token_line
 		let selector_column = this.lexer.token_column
 
-		// Consume tokens until we hit '{'
 		let last_end = this.lexer.token_end
-		while (!this.is_eof() && this.peek_type() !== TOKEN_LEFT_BRACE) {
-			last_end = this.lexer.token_end
+		if (this.peek_type() !== TOKEN_LEFT_BRACE) {
+			// Fast-forward to the next unquoted '{' (or EOF) without fully tokenizing
+			// everything in between — SelectorParser below re-tokenizes this exact span
+			// properly to build the AST, so this coarse pass only needs the boundary.
+			this.lexer.skip_to_unquoted(CHAR_LEFT_BRACE)
+			last_end = skip_whitespace_and_comments_backward(this.source, this.lexer.pos, selector_start)
+			// Tokenize the '{' (or EOF) so peek_type()/token_* reflect it for the caller,
+			// same as the old token-by-token loop would leave behind.
 			this.next_token()
 		}
 
diff --git a/src/string-utils.ts b/src/string-utils.ts
index 506c03d..d6b04f4 100644
--- a/src/string-utils.ts
+++ b/src/string-utils.ts
@@ -22,6 +22,12 @@ export const CHAR_DOLLAR = 0x24 // $
 export const CHAR_CARET = 0x5e // ^
 export const CHAR_COLON = 0x3a // :
 export const CHAR_LESS_THAN = 0x3c // <
+export const CHAR_LEFT_BRACE = 0x7b // {
+export const CHAR_SEMICOLON = 0x3b // ;
+export const CHAR_RIGHT_BRACE = 0x7d // }
+export const CHAR_LEFT_PAREN = 0x28 // (
+export const CHAR_RIGHT_PAREN = 0x29 // )
+export const CHAR_EXCLAMATION = 0x21 // !
 
 /**
  * Check if a character code is whitespace (space, tab, newline, CR, or FF)
diff --git a/src/tokenize.ts b/src/tokenize.ts
index f61fd1c..d07303e 100644
--- a/src/tokenize.ts
+++ b/src/tokenize.ts
@@ -74,6 +74,32 @@ const CHAR_CARRIAGE_RETURN = 0x0d // \r
 const CHAR_LINE_FEED = 0x0a // \n
 const CHAR_FORM_FEED = 0x0c // \f
 
+// Characters skip_to_unquoted() needs to stop and inspect: quotes (start a string), '/'
+// (might start a comment), and newlines (need line/column tracking). Built once and shared
+// across all skip_to_unquoted calls regardless of the (dynamic) target character being
+// searched for, so the ordinary-character fast path is one table lookup, same cost as
+// consume_ident_or_function's inner loop.
+const SKIP_SCAN_INTERESTING = new Uint8Array(128)
+SKIP_SCAN_INTERESTING[CHAR_DOUBLE_QUOTE] = 1
+SKIP_SCAN_INTERESTING[CHAR_SINGLE_QUOTE] = 1
+SKIP_SCAN_INTERESTING[CHAR_FORWARD_SLASH] = 1
+SKIP_SCAN_INTERESTING[CHAR_LINE_FEED] = 1
+SKIP_SCAN_INTERESTING[CHAR_CARRIAGE_RETURN] = 1
+SKIP_SCAN_INTERESTING[CHAR_FORM_FEED] = 1
+
+// Extra characters skip_to_declaration_stop() reports back to its caller (on top of the
+// string/comment/newline handling shared with skip_to_unquoted above): statement/block/
+// paren delimiters and '!' (for !important detection). parse-declaration.ts decides what
+// each one means (paren depth, !important, "actually a nested rule") - this table only
+// flags which characters are worth stopping for.
+const DECL_VALUE_STOP = new Uint8Array(128)
+DECL_VALUE_STOP[CHAR_SEMICOLON] = 1
+DECL_VALUE_STOP[CHAR_LEFT_BRACE] = 1
+DECL_VALUE_STOP[CHAR_RIGHT_BRACE] = 1
+DECL_VALUE_STOP[CHAR_LEFT_PAREN] = 1
+DECL_VALUE_STOP[CHAR_RIGHT_PAREN] = 1
+DECL_VALUE_STOP[CHAR_EXCLAMATION] = 1
+
 export interface LexerPosition {
 	pos: number
 	line: number
@@ -223,35 +249,7 @@ export class Lexer {
 				this.pos + 1 < source_length &&
 				source.charCodeAt(this.pos + 1) === CHAR_ASTERISK
 			) {
-				let comment_start = start
-				let comment_line = start_line
-				let comment_column = start_column
-
-				// Neither / nor * are newlines — safe to skip without newline tracking
-				this.pos += 2
-
-				// Use native string search to find */ — dramatically faster than a JS loop
-				// for typical comment bodies (SIMD-accelerated in V8).
-				let end_idx = source.indexOf('*/', this.pos)
-				if (end_idx < 0) {
-					// Unterminated comment: scan tail for newlines, advance to EOF
-					this._scan_newlines(this.pos, source_length)
-					this.pos = source_length
-				} else {
-					this._scan_newlines(this.pos, end_idx)
-					this.pos = end_idx + 2
-				}
-
-				if (this.on_comment) {
-					this.on_comment({
-						start: comment_start,
-						end: this.pos,
-						length: this.pos - comment_start,
-						line: comment_line,
-						column: comment_column,
-					})
-				}
-
+				this._skip_comment()
 				// Loop instead of recursing — eliminates stack frame overhead
 				continue
 			}
@@ -390,6 +388,179 @@ export class Lexer {
 		}
 	}
 
+	// Skip a comment (/* ... */), firing on_comment if set. Caller must have already
+	// verified this.pos points at '/' immediately followed by '*'. Shared by
+	// next_token_fast's hot loop and skip_to_unquoted, so both stay in sync by construction.
+	private _skip_comment(): void {
+		const source = this.source
+		const source_length = source.length
+		let comment_start = this.pos
+		let comment_line = this._line
+		let comment_column = this.pos - this._line_offset + 1
+
+		// Neither / nor * are newlines — safe to skip without newline tracking
+		this.pos += 2
+
+		// Use native string search to find */ — dramatically faster than a JS loop
+		// for typical comment bodies (SIMD-accelerated in V8).
+		let end_idx = source.indexOf('*/', this.pos)
+		if (end_idx < 0) {
+			// Unterminated comment: scan tail for newlines, advance to EOF
+			this._scan_newlines(this.pos, source_length)
+			this.pos = source_length
+		} else {
+			this._scan_newlines(this.pos, end_idx)
+			this.pos = end_idx + 2
+		}
+
+		if (this.on_comment) {
+			this.on_comment({
+				start: comment_start,
+				end: this.pos,
+				length: this.pos - comment_start,
+				line: comment_line,
+				column: comment_column,
+			})
+		}
+	}
+
+	/**
+	 * Fast-forward to the next occurrence of `target` (a char code) that isn't inside a
+	 * string or comment, WITHOUT fully tokenizing everything in between — unlike
+	 * next_token_fast, this never dispatches through consume_ident_or_function/consume_number/
+	 * etc. for ordinary characters. Used by callers that only need to know *where a
+	 * construct ends* (a selector's `{`, a value's `;`), not the tokens inside it — the
+	 * detailed sub-parser for that construct re-tokenizes the span properly afterward.
+	 *
+	 * Strings are skipped via consume_string itself (not reimplemented here), so escape
+	 * sequences, hex escapes, escaped newlines, and bad/unterminated strings are handled
+	 * identically to real tokenization. Comments are skipped via the same _skip_comment
+	 * used by next_token_fast, so on_comment still fires for comments encountered here.
+	 *
+	 * The ordinary-character fast path is a single table lookup (SKIP_SCAN_INTERESTING),
+	 * same shape as consume_ident_or_function's inner loop, so this doesn't cost more per
+	 * character than the full tokenizer would for a run of identifier/delimiter chars —
+	 * the savings come from skipping token classification and make_token entirely for them.
+	 *
+	 * Leaves this.pos at target's position (returns true) or at source_length (returns
+	 * false); this._line/_line_offset stay accurate for a subsequent seek/next_token_fast.
+	 */
+	skip_to_unquoted(target: number): boolean {
+		const source = this.source
+		const source_length = source.length
+		let pos = this.pos
+
+		while (pos < source_length) {
+			let ch = source.charCodeAt(pos)
+
+			if (ch === target) {
+				this.pos = pos
+				return true
+			}
+
+			if (ch >= 128 || SKIP_SCAN_INTERESTING[ch] === 0) {
+				pos++
+				continue
+			}
+
+			if (ch === CHAR_DOUBLE_QUOTE || ch === CHAR_SINGLE_QUOTE) {
+				// start_line/start_column are only used for the token this returns, which we
+				// discard - consume_string's side effects on this.pos/_line/_line_offset are
+				// what we actually want, and those are correct regardless of what we pass here.
+				this.pos = pos
+				this.consume_string(ch, this._line, 1)
+				pos = this.pos
+				continue
+			}
+
+			if (ch === CHAR_FORWARD_SLASH) {
+				if (pos + 1 < source_length && source.charCodeAt(pos + 1) === CHAR_ASTERISK) {
+					this.pos = pos
+					this._skip_comment()
+					pos = this.pos
+				} else {
+					pos++
+				}
+				continue
+			}
+
+			// Newline (LF, CR, or FF) - track line/column even though we're not building tokens
+			pos++
+			if (
+				ch === CHAR_CARRIAGE_RETURN &&
+				pos < source_length &&
+				source.charCodeAt(pos) === CHAR_LINE_FEED
+			) {
+				pos++
+			}
+			this._line++
+			this._line_offset = pos
+		}
+
+		this.pos = pos
+		return false
+	}
+
+	/**
+	 * Fast-forward to the next occurrence of one of `; { } ( ) !` that isn't inside a
+	 * string or comment, WITHOUT fully tokenizing the ordinary content in between - same
+	 * approach as skip_to_unquoted, specialized for the declaration-value boundary scan in
+	 * parse-declaration.ts (which needs several different stop characters: paren depth
+	 * tracking needs both `(` and `)`, statement end is `;`/`}`, and `!` flags a possible
+	 * !important). That caller decides what each returned character means; this method only
+	 * locates the next one.
+	 *
+	 * Returns the stop character's code, leaving this.pos at its position. Returns 0 (and
+	 * leaves this.pos at `end`) if none is found first.
+	 */
+	skip_to_declaration_stop(end: number): number {
+		const source = this.source
+		let pos = this.pos
+
+		while (pos < end) {
+			let ch = source.charCodeAt(pos)
+
+			if (ch < 128 && DECL_VALUE_STOP[ch] !== 0) {
+				this.pos = pos
+				return ch
+			}
+
+			if (ch >= 128 || SKIP_SCAN_INTERESTING[ch] === 0) {
+				pos++
+				continue
+			}
+
+			if (ch === CHAR_DOUBLE_QUOTE || ch === CHAR_SINGLE_QUOTE) {
+				this.pos = pos
+				this.consume_string(ch, this._line, 1)
+				pos = this.pos
+				continue
+			}
+
+			if (ch === CHAR_FORWARD_SLASH) {
+				if (pos + 1 < end && source.charCodeAt(pos + 1) === CHAR_ASTERISK) {
+					this.pos = pos
+					this._skip_comment()
+					pos = this.pos
+				} else {
+					pos++
+				}
+				continue
+			}
+
+			// Newline (LF, CR, or FF) - track line/column even though we're not building tokens
+			pos++
+			if (ch === CHAR_CARRIAGE_RETURN && pos < end && source.charCodeAt(pos) === CHAR_LINE_FEED) {
+				pos++
+			}
+			this._line++
+			this._line_offset = pos
+		}
+
+		this.pos = pos
+		return 0
+	}
+
 	consume_whitespace(start_line: number, start_column: number): TokenType {
 		const source = this.source
 		const source_length = source.length

From ebcce39d1960b993863f82a2685de45889d351bb Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Wed, 29 Jul 2026 17:39:48 +0000
Subject: [PATCH 4/4] Trim newly added comments to match the codebase's terser
 style

The double-tokenization elimination work (previous commit) and the
tokenizer dispatch/hoisting commits erred toward over-explaining in
comments compared to the rest of the file. Cut them down while keeping
the essential why, no behavior change.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017XjfDK7or6VrnJ41vBhxrK
---
 src/parse-declaration.ts | 53 ++++++++----------------
 src/parse.ts             |  8 ++--
 src/tokenize.ts          | 88 ++++++++++++++--------------------------
 3 files changed, 51 insertions(+), 98 deletions(-)

diff --git a/src/parse-declaration.ts b/src/parse-declaration.ts
index 72191ca..245a9f5 100644
--- a/src/parse-declaration.ts
+++ b/src/parse-declaration.ts
@@ -149,11 +149,9 @@ export class DeclarationParser {
 			lexer.restore_position(has_delimiter_prefix ? initial_saved : saved)
 			return null
 		}
-		// Skip whitespace/comments after ':' WITHOUT tokenizing the first value token - the
-		// raw scan below (skip_to_declaration_stop) needs to see every character of the value
-		// uniformly, including what would otherwise become an already-consumed first token
-		// (e.g. a leading "calc(" - its '(' must reach the paren-depth tracking below, which
-		// it wouldn't if next_token_fast had already tokenized past it here).
+		// Skip whitespace after ':' without tokenizing the value's first token - the raw scan
+		// below needs to see every value character itself (e.g. a leading "calc("'s '(' must
+		// reach the paren-depth tracking, not be pre-consumed here).
 		lexer.pos = lexer.token_end // move past ':' (already at this position, but be explicit)
 		lexer.skip_whitespace_in_range(this.source.length)
 
@@ -170,8 +168,7 @@ export class DeclarationParser {
 		this.arena.set_content_start_delta(declaration, 0)
 		this.arena.set_content_length(declaration, prop_end - prop_start)
 
-		// Track value start (after colon, skipping whitespace) - CRITICAL: Capture line/column
-		// for value parsing. Lexer is now positioned at the first value character (untokenized).
+		// Value start (after colon/whitespace) - lexer is positioned at the first, untokenized char.
 		let value_start = lexer.pos
 		let value_start_line = lexer.line
 		let value_start_column = lexer.column
@@ -183,14 +180,10 @@ export class DeclarationParser {
 		// Track parenthesis depth to handle semicolons inside functions (e.g., url(data:image/png;base64,...))
 		let paren_depth = 0
 
-		// Fast-forward through the value using a raw character scan for the exact stop
-		// points the token-by-token loop used to check (paren depth, ';'/'}' at depth zero,
-		// an unparenthesized '{', and '!' for !important) - the ordinary content in between
-		// doesn't need full tokenization here, since ValueNodeParser re-tokenizes the
-		// resulting span properly afterward.
-		// NOTE: every exit is an explicit break/return - the "ran out of input" case is only
-		// detected via skip_to_declaration_stop returning 0, not via a loop condition, since a
-		// paren_depth adjustment can land exactly on `end` without going through that path.
+		// Raw character scan for the value's stop points (paren depth, ';'/'}' at depth zero,
+		// '{', '!') - content in between doesn't need tokenizing since ValueNodeParser
+		// re-tokenizes the span afterward. Loop relies only on explicit break/return, since a
+		// paren_depth adjustment can land exactly on `end` without the "ran out" case firing.
 		while (true) {
 			let stop_ch = lexer.skip_to_declaration_stop(end)
 
@@ -207,36 +200,30 @@ export class DeclarationParser {
 
 			if (stop_ch === CHAR_SEMICOLON && paren_depth === 0) {
 				value_end = skip_whitespace_and_comments_backward(this.source, lexer.pos, value_start)
-				// Tokenize ';' so token_type reflects it, matching the state the old
-				// token-by-token loop left behind right when its condition saw TOKEN_SEMICOLON
-				// (already tokenized, not yet consumed) - the unchanged code below relies on it.
+				// Tokenize ';' so token_type reflects it, matching what the old loop left behind.
 				lexer.next_token_fast(false)
 				break
 			}
 
 			if (stop_ch === CHAR_RIGHT_BRACE && paren_depth === 0) {
 				if (lexer.pos === value_start) {
-					// Degenerate case: colon directly followed by the block's closing brace
-					// with no value content at all (e.g. "color:}"). Replicates a quirk of the
-					// original token-based scan: the pre-tokenized "first value token" being
-					// '}' itself meant its end position became last_end before the per-token
-					// loop ever got a chance to run.
+					// Degenerate case: "color:}" (no value content). Replicates a quirk of the old
+					// pre-tokenized scan, where '}' itself became the "first value token" and its
+					// end position became last_end before the per-token loop ever ran.
 					last_end = lexer.pos + 1
 					value_end = value_start
 				} else {
 					last_end = skip_whitespace_and_comments_backward(this.source, lexer.pos, value_start)
 					value_end = last_end
 				}
-				// Tokenize '}' so the caller's peek_type() sees it correctly - this declaration
-				// ends at a block boundary with no trailing semicolon, and '}' isn't consumed
-				// here (the enclosing block's own loop needs to see it to know it's done).
+				// Tokenize '}' (without consuming it) so peek_type() sees it - the enclosing
+				// block's own loop needs to see '}' to know it's done.
 				lexer.next_token_fast(false)
 				break
 			}
 
 			if (stop_ch === CHAR_LEFT_BRACE) {
-				// This is actually a style rule, not a declaration - '{' can appear at any
-				// paren depth and always bails out, regardless of depth.
+				// Actually a style rule, not a declaration - '{' bails out at any paren depth.
 				lexer.restore_position(saved)
 				return null
 			}
@@ -245,8 +232,7 @@ export class DeclarationParser {
 				// Mark end of value before !important
 				value_end = lexer.pos
 				lexer.pos++ // consume '!'
-				// Check if next token is an identifier (doesn't verify it's literally
-				// "important" - matches the pre-existing behavior this replaces)
+				// Check if next token is an identifier (doesn't verify it's literally "important")
 				let next_type = lexer.next_token_fast(true) // skip whitespace
 				if (next_type === TOKEN_IDENT) {
 					has_important = true
@@ -255,11 +241,8 @@ export class DeclarationParser {
 					break
 				}
 				// '!' wasn't followed by an identifier - treat the already-peeked token as
-				// ordinary content. lexer.pos is already right after it (tokenizing to peek
-				// advances past it), so the raw scan below picks up from there naturally - an
-				// extra next_token_fast here would blindly consume whatever comes next (e.g. a
-				// block's closing '}') without the scan ever getting a chance to treat it as a
-				// stop condition.
+				// ordinary content; lexer.pos is already past it, so the raw scan picks up
+				// naturally (an extra next_token_fast here would swallow e.g. a trailing '}').
 				last_end = lexer.token_end
 				value_end = last_end
 				continue
diff --git a/src/parse.ts b/src/parse.ts
index bbd5ec5..d29627c 100644
--- a/src/parse.ts
+++ b/src/parse.ts
@@ -307,13 +307,11 @@ export class Parser {
 
 		let last_end = this.lexer.token_end
 		if (this.peek_type() !== TOKEN_LEFT_BRACE) {
-			// Fast-forward to the next unquoted '{' (or EOF) without fully tokenizing
-			// everything in between — SelectorParser below re-tokenizes this exact span
-			// properly to build the AST, so this coarse pass only needs the boundary.
+			// Fast-forward to the unquoted '{' (or EOF) — SelectorParser below re-tokenizes
+			// this span properly, so this coarse pass only needs the boundary.
 			this.lexer.skip_to_unquoted(CHAR_LEFT_BRACE)
 			last_end = skip_whitespace_and_comments_backward(this.source, this.lexer.pos, selector_start)
-			// Tokenize the '{' (or EOF) so peek_type()/token_* reflect it for the caller,
-			// same as the old token-by-token loop would leave behind.
+			// Tokenize it so peek_type()/token_* reflect it, as the old loop would leave behind.
 			this.next_token()
 		}
 
diff --git a/src/tokenize.ts b/src/tokenize.ts
index d07303e..96b1a6f 100644
--- a/src/tokenize.ts
+++ b/src/tokenize.ts
@@ -74,11 +74,9 @@ const CHAR_CARRIAGE_RETURN = 0x0d // \r
 const CHAR_LINE_FEED = 0x0a // \n
 const CHAR_FORM_FEED = 0x0c // \f
 
-// Characters skip_to_unquoted() needs to stop and inspect: quotes (start a string), '/'
-// (might start a comment), and newlines (need line/column tracking). Built once and shared
-// across all skip_to_unquoted calls regardless of the (dynamic) target character being
-// searched for, so the ordinary-character fast path is one table lookup, same cost as
-// consume_ident_or_function's inner loop.
+// Characters skip_to_unquoted()/skip_to_declaration_stop() must stop and inspect: quotes,
+// comment-starting '/', and newlines. One shared table keeps the ordinary-char fast path
+// to a single lookup, same cost as consume_ident_or_function's inner loop.
 const SKIP_SCAN_INTERESTING = new Uint8Array(128)
 SKIP_SCAN_INTERESTING[CHAR_DOUBLE_QUOTE] = 1
 SKIP_SCAN_INTERESTING[CHAR_SINGLE_QUOTE] = 1
@@ -87,11 +85,9 @@ SKIP_SCAN_INTERESTING[CHAR_LINE_FEED] = 1
 SKIP_SCAN_INTERESTING[CHAR_CARRIAGE_RETURN] = 1
 SKIP_SCAN_INTERESTING[CHAR_FORM_FEED] = 1
 
-// Extra characters skip_to_declaration_stop() reports back to its caller (on top of the
-// string/comment/newline handling shared with skip_to_unquoted above): statement/block/
-// paren delimiters and '!' (for !important detection). parse-declaration.ts decides what
-// each one means (paren depth, !important, "actually a nested rule") - this table only
-// flags which characters are worth stopping for.
+// Stop characters skip_to_declaration_stop() reports back: statement/block/paren
+// delimiters plus '!' for !important. The caller (parse-declaration.ts) decides what
+// each one means; this table only flags which are worth stopping for.
 const DECL_VALUE_STOP = new Uint8Array(128)
 DECL_VALUE_STOP[CHAR_SEMICOLON] = 1
 DECL_VALUE_STOP[CHAR_LEFT_BRACE] = 1
@@ -169,9 +165,8 @@ export class Lexer {
 		// Outer loop replaces comment recursion: after consuming a comment, continue
 		// to find the actual next token instead of making a recursive call.
 		while (true) {
-			// Inline whitespace skip — avoids method call + duplicate charCodeAt per char.
-			// Hoisted to locals (synced back once) instead of touching this.pos/_line/_line_offset
-			// on every character — most runs are plain spaces that never hit the newline branch.
+			// Inline whitespace skip, hoisted to locals (synced back once) instead of touching
+			// this.pos/_line/_line_offset per char — most runs are plain spaces.
 			if (skip_whitespace) {
 				let pos = this.pos
 				let line = this._line
@@ -254,21 +249,16 @@ export class Lexer {
 				continue
 			}
 
-			// Identifier or function — checked early: across real-world CSS this is the single
-			// most common non-whitespace token (~15-20%+ of all tokens), well ahead of numbers,
-			// hashes, at-keywords, strings, and the (essentially extinct in modern CSS) CDO/CDC
-			// tokens that used to precede it here. Measured via profiling + token frequency
-			// analysis on bootstrap.css/tailwind.css — see PR discussion for numbers.
+			// Identifier or function — checked first: the single most common non-whitespace
+			// token (~15-20% of all tokens), ahead of numbers/hashes/at-keywords/strings and
+			// the now-extinct CDO/CDC tokens that used to precede it (profiled on bootstrap.css/tailwind.css).
 			if (is_ident_start(ch)) {
 				return this.consume_ident_or_function(start_line, start_column)
 			}
 
-			// Hyphen-led tokens: --custom-property/-webkit-prefix (identifier), -5px (signed
-			// number), or the legacy --> CDC token. Consolidated into one block (single charCodeAt
-			// for the lookahead char) instead of three separate `ch === CHAR_HYPHEN` checks.
-			// NOTE: CDC must be checked first — "-->" would otherwise be misread as identifier "--"
-			// followed by a stray ">" delimiter, since -- is itself valid identifier-start (custom
-			// properties).
+			// Hyphen-led tokens: --custom-property (identifier), -5px (signed number), or the
+			// legacy --> CDC token — consolidated into one block sharing a single lookahead read.
+			// CDC must be checked first: "-->" would otherwise read as identifier "--" plus ">".
 			if (ch === CHAR_HYPHEN) {
 				let next = this.pos + 1 < source_length ? source.charCodeAt(this.pos + 1) : 0
 
@@ -346,8 +336,7 @@ export class Lexer {
 				return this.consume_hash(start_line, start_column)
 			}
 
-			// CDO: <!-- (essentially extinct in modern CSS — checked last, right before the
-			// default delimiter fallback; a lone "<" that isn't CDO just falls through to DELIM)
+			// CDO: <!-- (extinct in modern CSS — checked last; a lone "<" falls through to DELIM)
 			if (ch === CHAR_LESS_THAN && this.pos + 3 < source_length) {
 				if (
 					source.charCodeAt(this.pos + 1) === CHAR_EXCLAMATION &&
@@ -388,9 +377,8 @@ export class Lexer {
 		}
 	}
 
-	// Skip a comment (/* ... */), firing on_comment if set. Caller must have already
-	// verified this.pos points at '/' immediately followed by '*'. Shared by
-	// next_token_fast's hot loop and skip_to_unquoted, so both stay in sync by construction.
+	// Skip a comment (/* ... */), firing on_comment if set. Caller must have verified
+	// this.pos is at '/*'. Shared by next_token_fast and the skip_to_* scans below.
 	private _skip_comment(): void {
 		const source = this.source
 		const source_length = source.length
@@ -425,25 +413,16 @@ export class Lexer {
 	}
 
 	/**
-	 * Fast-forward to the next occurrence of `target` (a char code) that isn't inside a
-	 * string or comment, WITHOUT fully tokenizing everything in between — unlike
-	 * next_token_fast, this never dispatches through consume_ident_or_function/consume_number/
-	 * etc. for ordinary characters. Used by callers that only need to know *where a
-	 * construct ends* (a selector's `{`, a value's `;`), not the tokens inside it — the
-	 * detailed sub-parser for that construct re-tokenizes the span properly afterward.
+	 * Fast-forward to the next `target` char that isn't inside a string or comment, without
+	 * fully tokenizing what's in between — used by callers that only need to know *where a
+	 * construct ends* (e.g. a selector's `{`), since the sub-parser for that construct
+	 * re-tokenizes the span properly afterward.
 	 *
-	 * Strings are skipped via consume_string itself (not reimplemented here), so escape
-	 * sequences, hex escapes, escaped newlines, and bad/unterminated strings are handled
-	 * identically to real tokenization. Comments are skipped via the same _skip_comment
-	 * used by next_token_fast, so on_comment still fires for comments encountered here.
+	 * Strings go through consume_string and comments through _skip_comment, so escapes and
+	 * on_comment callbacks behave identically to real tokenization. Ordinary characters cost
+	 * one SKIP_SCAN_INTERESTING lookup each, same as consume_ident_or_function's inner loop.
 	 *
-	 * The ordinary-character fast path is a single table lookup (SKIP_SCAN_INTERESTING),
-	 * same shape as consume_ident_or_function's inner loop, so this doesn't cost more per
-	 * character than the full tokenizer would for a run of identifier/delimiter chars —
-	 * the savings come from skipping token classification and make_token entirely for them.
-	 *
-	 * Leaves this.pos at target's position (returns true) or at source_length (returns
-	 * false); this._line/_line_offset stay accurate for a subsequent seek/next_token_fast.
+	 * Returns true with this.pos at target, or false with this.pos at source_length.
 	 */
 	skip_to_unquoted(target: number): boolean {
 		const source = this.source
@@ -464,9 +443,7 @@ export class Lexer {
 			}
 
 			if (ch === CHAR_DOUBLE_QUOTE || ch === CHAR_SINGLE_QUOTE) {
-				// start_line/start_column are only used for the token this returns, which we
-				// discard - consume_string's side effects on this.pos/_line/_line_offset are
-				// what we actually want, and those are correct regardless of what we pass here.
+				// The returned token is discarded — only consume_string's pos/line side effects matter.
 				this.pos = pos
 				this.consume_string(ch, this._line, 1)
 				pos = this.pos
@@ -502,16 +479,11 @@ export class Lexer {
 	}
 
 	/**
-	 * Fast-forward to the next occurrence of one of `; { } ( ) !` that isn't inside a
-	 * string or comment, WITHOUT fully tokenizing the ordinary content in between - same
-	 * approach as skip_to_unquoted, specialized for the declaration-value boundary scan in
-	 * parse-declaration.ts (which needs several different stop characters: paren depth
-	 * tracking needs both `(` and `)`, statement end is `;`/`}`, and `!` flags a possible
-	 * !important). That caller decides what each returned character means; this method only
-	 * locates the next one.
+	 * Same approach as skip_to_unquoted, but stops at any of `; { } ( ) !` — the stop
+	 * characters parse-declaration.ts's value scan needs (paren depth, statement end,
+	 * !important). The caller decides what each one means; this only locates the next one.
 	 *
-	 * Returns the stop character's code, leaving this.pos at its position. Returns 0 (and
-	 * leaves this.pos at `end`) if none is found first.
+	 * Returns the stop character's code (this.pos left at it), or 0 at `end` if none found.
 	 */
 	skip_to_declaration_stop(end: number): number {
 		const source = this.source