From 3269ba2962f5b6d88df25058837c14459d69d06b Mon Sep 17 00:00:00 2001 From: Ryan Atkinson Date: Wed, 2 Sep 2026 22:20:55 -0400 Subject: [PATCH] perf: the CR fold's pass takes the line verdict, so a document that folds is walked once --- CLAUDE.md | 4 +- README.md | 2 +- crates/tsv_cli/src/cli/format_source.rs | 14 +- crates/tsv_css/CLAUDE.md | 4 +- crates/tsv_css/src/lib.rs | 19 +- crates/tsv_css/src/printer/mod.rs | 32 ++- crates/tsv_debug/src/audit/census.rs | 2 +- .../tsv_debug/src/cli/commands/scan_audit.rs | 5 - crates/tsv_ffi/CLAUDE.md | 2 +- crates/tsv_ffi/src/lib.rs | 6 +- crates/tsv_lang/CLAUDE.md | 4 +- crates/tsv_lang/src/printing.rs | 264 ++++++++++++++++-- crates/tsv_napi/CLAUDE.md | 2 +- crates/tsv_napi/src/lib.rs | 6 +- crates/tsv_svelte/CLAUDE.md | 4 +- crates/tsv_svelte/src/lib.rs | 19 +- crates/tsv_svelte/src/printer/mod.rs | 65 ++++- crates/tsv_svelte_compile/src/lib.rs | 2 +- crates/tsv_ts/CLAUDE.md | 4 +- crates/tsv_ts/src/lib.rs | 55 +++- crates/tsv_wasm/src/lib.rs | 6 +- docs/architecture.md | 6 +- docs/performance.md | 43 +++ 23 files changed, 478 insertions(+), 92 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 7fcbd654c..6d8e0a551 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -832,7 +832,9 @@ acorn / Svelte / `parseCss` over the author's own bytes. **Every parse-then-form point folds `` / `` to `` before it parses** (`tsv_lang::printing::normalize_carriage_returns` — each language crate's `format_str`, the CLI's `format_source`, each binding's format export, `canonicalize_js`), so tsv's output is -LF-only even inside the regions it copies verbatim. Ahead of the parse is the only place +LF-only even inside the regions it copies verbatim. The fold's one pass also takes the +folded document's line verdict (`FoldedSource`), which each crate's `format_folded_in` +hands its printer, so a document that folds is walked once. Ahead of the parse is the only place that answers it once: the printers ask "where are the lines?" in several places that split on `'\n'` alone, and folding the finished string instead leaves those disagreeing with the output — the same document then formats two ways on two passes. `` / `` are diff --git a/README.md b/README.md index f175904b2..91fe38ca5 100644 --- a/README.md +++ b/README.md @@ -289,7 +289,7 @@ tsv/ Each language crate exports a consistent API: - `parse(source, arena) -> Result` — the AST allocates into the caller's `bumpalo` arena (the bindings reuse a per-thread arena across calls via `tsv_arena`) -- `format(ast, source) -> String` — plus `format_in(ast, source, doc_arena)`, the same formatter writing through a reusable doc arena for the bindings' hot loop, and `format_str(source)`, the parse+format one-shot +- `format(ast, source) -> String` — plus `format_in(ast, source, doc_arena)`, the same formatter writing through a reusable doc arena for the bindings' hot loop, `format_folded_in(ast, &folded, doc_arena)` for a caller that folded the source's line terminators itself, and `format_str(source)`, the parse+format one-shot - `convert_ast_json_bytes(ast, source) -> Vec` — the wire JSON, emitted directly from the internal AST, with `convert_ast_json_string`/`convert_ast_json` wrappers and span-only `_no_locations` variants alongside (default-on `convert` cargo feature; turn off for parse+format-only builds) For more details see [CLAUDE.md](CLAUDE.md). diff --git a/crates/tsv_cli/src/cli/format_source.rs b/crates/tsv_cli/src/cli/format_source.rs index 974d81b8c..c593e2e95 100644 --- a/crates/tsv_cli/src/cli/format_source.rs +++ b/crates/tsv_cli/src/cli/format_source.rs @@ -68,18 +68,20 @@ pub fn format_source_in_with_goal( // printer never sees a `` and the doc-build's line splits agree with the output // about where the lines are. See `tsv_lang::printing::normalize_carriage_returns`; the // `parse` command deliberately skips it, its offsets being a drop-in contract over the - // author's own bytes. Borrowed unchanged on a source with no ``. - let source = tsv_lang::printing::normalize_carriage_returns(source); - let source = source.as_ref(); + // author's own bytes. Borrowed unchanged on a source with no ``, and the fold's + // pass is the document's line verdict too — the `format_folded_in` siblings take it + // rather than walking the source again. + let folded = tsv_lang::printing::normalize_carriage_returns(source); + let source = folded.text(); match parser_type { ParserType::Svelte => tsv_svelte::parse(source, arena) - .map(|ast| tsv_svelte::format_in(&ast, source, doc_arena)) + .map(|ast| tsv_svelte::format_folded_in(&ast, &folded, doc_arena)) .map_err(|e| e.to_string()), ParserType::Css => tsv_css::parse(source, arena) - .map(|ast| tsv_css::format_in(&ast, source, doc_arena)) + .map(|ast| tsv_css::format_folded_in(&ast, &folded, doc_arena)) .map_err(|e| e.to_string()), ParserType::TypeScript => tsv_ts::parse_with_goal(source, goal, arena) - .map(|ast| tsv_ts::format_in(&ast, source, doc_arena)) + .map(|ast| tsv_ts::format_folded_in(&ast, &folded, doc_arena)) .map_err(|e| e.to_string()), } } diff --git a/crates/tsv_css/CLAUDE.md b/crates/tsv_css/CLAUDE.md index fb5d64ccf..0721b05af 100644 --- a/crates/tsv_css/CLAUDE.md +++ b/crates/tsv_css/CLAUDE.md @@ -15,8 +15,8 @@ Standard `ast/lexer/parser/printer` crate layout — see [root CLAUDE.md §Proje **Standalone** (top-level CSS files): - `parse(source, arena: &'arena Bump) -> Result>` — parse a full CSS file. The internal AST is bump-arena-allocated (caller-owns-`Bump`); `CssStyleSheet<'arena>` borrows from it. Matches `tsv_ts`/`tsv_svelte`'s signature for the shared `lang_bindings!` macro + CLI/FFI/WASM callers. -- `format_str(source) -> Result` — parse + `format` fused (applies the format path's `` fold ahead of the parse), the same twin every language crate has -- `format(&stylesheet, source) -> String` — format with default config; `format_in(&stylesheet, source, &DocArena)` is the same into a caller-provided doc arena (multi-file drivers reuse one via `DocArena::reset()`; `format` is the fresh-arena wrapper) +- `format_str(source) -> Result` — parse + `format` fused (applies the format path's `` fold ahead of the parse and formats through `format_folded_in`), the same twin every language crate has +- `format(&stylesheet, source) -> String` — format with default config; `format_in(&stylesheet, source, &DocArena)` is the same into a caller-provided doc arena (multi-file drivers reuse one via `DocArena::reset()`; `format` is the fresh-arena wrapper); `format_folded_in(&stylesheet, &FoldedSource, &DocArena)` is `format_in` for a caller that folded the source itself — identical output, the line verdict taken from the fold's pass (what the CLI, the bindings and `format_str` call) - `convert_ast_json_bytes(&stylesheet, source) -> Vec` / `convert_ast_json_string(...) -> String` — the **sole emission path** (bytes: FFI/CLI non-pretty; string: the same bytes plus one output UTF-8 validation, for WASM `JSON.parse` / N-API): the writer (`ast/convert/write.rs`) walks the *internal* AST once and emits the wire JSON directly into a pre-sized buffer (`tsv_lang::estimated_json_capacity`), never materializing a typed public tree, fusing byte→char offset translation into the walk (each position through a `ByteToCharMap`; identity on ASCII). Matches Svelte's `parseCss()` JSON shape, reusing the raw-source reconstruction helpers in `ast/convert/mod.rs` (`strip_css_comments`, `split_declaration_svelte_compat`, `raw_selector_name`, …). Gated against the canonical `parseCss` `expected.json` and `corpus:compare:parse --multibyte-only` - `convert_ast_json(&stylesheet, source) -> serde_json::Value` — a thin wrapper (`serde_json::from_slice(&convert_ast_json_bytes(...))`) for the `Value` consumers (the CLI's `--pretty`, the fixture gate); not an independent conversion - `convert_ast_json_bytes_no_locations(...)` / `convert_ast_json_string_no_locations(...)` — the span-only variant, present only for parity with the TS/Svelte writers and the uniform `lang_bindings!` macro. `parseCss` emits **no** per-node `loc`, so the CSS wire is already offset-only — these are **exact aliases** of the plain functions (a documented no-op, not a distinct shape). The CLI/FFI/N-API/WASM `no-locations` surfaces therefore treat the option as inert for CSS — their generated CSS rows exist for macro uniformity and are byte-identical to the plain parse. diff --git a/crates/tsv_css/src/lib.rs b/crates/tsv_css/src/lib.rs index 48ed81681..630642be5 100644 --- a/crates/tsv_css/src/lib.rs +++ b/crates/tsv_css/src/lib.rs @@ -99,10 +99,11 @@ pub fn format_str(source: &str) -> Result { // The format path's line-terminator fold, ahead of the parse (see // `tsv_lang::printing::normalize_carriage_returns`); `parse` leaves the author's bytes // alone so its offsets stay a drop-in contract with `parseCss`'s. - let source = tsv_lang::printing::normalize_carriage_returns(source); + let folded = tsv_lang::printing::normalize_carriage_returns(source); let arena = bumpalo::Bump::new(); - let stylesheet = parse(&source, &arena)?; - Ok(format(&stylesheet, &source)) + let stylesheet = parse(folded.text(), &arena)?; + let doc_arena = tsv_lang::doc::arena::DocArena::for_source(folded.text()); + Ok(format_folded_in(&stylesheet, &folded, &doc_arena)) } /// Format into a caller-provided doc arena. @@ -119,6 +120,18 @@ pub fn format_in( printer::format_css_in(stylesheet, source, arena) } +/// [`format_in`] over a document the caller folded ahead of the parse +/// (`tsv_lang::printing::normalize_carriage_returns`) — the format entry points that fold +/// (the CLI, the bindings, [`format_str`]). Identical output; the document's line verdict +/// comes from the fold's own pass instead of a second walk of the source. +pub fn format_folded_in( + stylesheet: &CssStyleSheet<'_>, + folded: &tsv_lang::printing::FoldedSource<'_>, + arena: &tsv_lang::doc::arena::DocArena, +) -> String { + printer::format_css_folded_in(stylesheet, folded, arena) +} + /// Format an embedded CSS stylesheet into a caller-provided doc arena. /// /// `embed.base_indent_offset` seeds the indent so wrapped lines respect the host's diff --git a/crates/tsv_css/src/printer/mod.rs b/crates/tsv_css/src/printer/mod.rs index 9ae444574..695fc0df1 100644 --- a/crates/tsv_css/src/printer/mod.rs +++ b/crates/tsv_css/src/printer/mod.rs @@ -40,7 +40,7 @@ use tsv_lang::{ arena::{DocArena, DocId}, }, is_format_ignore_directive, - printing::{self, LineBreaks, LineTable}, + printing::{self, FoldedSource, LineBreaks, LineTable}, }; /// Printer state for building output @@ -1150,6 +1150,32 @@ pub(crate) fn format_css_in( stylesheet: &CssStyleSheet<'_>, source: &str, arena: &DocArena, +) -> String { + // The document's line verdict, over the arena-parked table (one warm table across a + // multi-file driver's files instead of a fresh Vec per file — filled only if a line + // question falls back to it). + let line_breaks = LineBreaks::new(source, arena.take_line_breaks_scratch()); + format_stylesheet(stylesheet, source, line_breaks, arena) +} + +/// [`format_css_in`] over a document the caller folded ahead of the parse: the line +/// verdict comes from the fold's own pass (`LineBreaks::of_folded`), not a second walk. +pub(crate) fn format_css_folded_in( + stylesheet: &CssStyleSheet<'_>, + folded: &FoldedSource<'_>, + arena: &DocArena, +) -> String { + let line_breaks = LineBreaks::of_folded(folded, arena.take_line_breaks_scratch()); + format_stylesheet(stylesheet, folded.text(), line_breaks, arena) +} + +/// The shared body of the two: register the stylesheet's comments, build the printer on +/// its line table, print, park the table. +fn format_stylesheet( + stylesheet: &CssStyleSheet<'_>, + source: &str, + line_breaks: LineBreaks<'_>, + arena: &DocArena, ) -> String { // The print-once comment ledger's expectation for this stylesheet — detached comments // plus in-block `CssBlockChild::Comment` AST nodes (diagnostic; see @@ -1157,10 +1183,6 @@ pub(crate) fn format_css_in( #[cfg(feature = "comment_check")] register_stylesheet_comments(stylesheet, source); - // The document's line verdict, over the arena-parked table (one warm table across a - // multi-file driver's files instead of a fresh Vec per file — filled only if a line - // question falls back to it). - let line_breaks = LineBreaks::new(source, arena.take_line_breaks_scratch()); let mut printer = Printer::new(arena, source, &stylesheet.comments, line_breaks.table()); printer.print_css_nodes(stylesheet.nodes); let output = printer.into_string(); diff --git a/crates/tsv_debug/src/audit/census.rs b/crates/tsv_debug/src/audit/census.rs index b7a6d9e73..061041490 100644 --- a/crates/tsv_debug/src/audit/census.rs +++ b/crates/tsv_debug/src/audit/census.rs @@ -157,7 +157,7 @@ pub(crate) fn comment_census(source: &str, parser: ParserType) -> CensusMultiset /// make `ab` and `ab` compare equal and blind the census to a real rewrite. `\r` /// left the trim class in the same step — after the fold there is none left to trim. fn normalize_interior(kind: CensusKind, raw: &str) -> String { - let raw = tsv_lang::printing::normalize_carriage_returns(raw); + let raw = tsv_lang::printing::normalize_carriage_returns(raw).into_text(); match kind { // Runs to end of line, so it has exactly one edge the printer touches. CensusKind::Line => raw.trim_end_matches(tsv_lang::is_js_whitespace).to_owned(), diff --git a/crates/tsv_debug/src/cli/commands/scan_audit.rs b/crates/tsv_debug/src/cli/commands/scan_audit.rs index 95195d35d..7644ea20c 100644 --- a/crates/tsv_debug/src/cli/commands/scan_audit.rs +++ b/crates/tsv_debug/src/cli/commands/scan_audit.rs @@ -142,11 +142,6 @@ const ALLOW: &[Allow] = &[ "if let Some(last_newline_pos) = s.rfind('\\n') {", "non-source", ), - ( - "tsv_lang/src/printing.rs", - "let Some(first) = source.find('\\r') else {", - "terminator-fold", - ), ( "tsv_lang/src/printing.rs", "while let Some(i) = rest.find('\\r') {", diff --git a/crates/tsv_ffi/CLAUDE.md b/crates/tsv_ffi/CLAUDE.md index 69cf0a139..c244b0a1f 100644 --- a/crates/tsv_ffi/CLAUDE.md +++ b/crates/tsv_ffi/CLAUDE.md @@ -6,7 +6,7 @@ Depends on `tsv_ts`, `tsv_css`, `tsv_svelte`. Sibling binding crates: [`tsv_wasm`](../tsv_wasm/) (WebAssembly) and [`tsv_napi`](../tsv_napi/) (N-API, the Node/Bun native path). This crate is the C-ABI path — consumers include Deno FFI, Python `ctypes`, and any other C-FFI host. Node/Bun use `tsv_napi` instead (no C-FFI glue); the per-thread arena reuse below is shared across all three bindings via the [`tsv_arena`](../tsv_arena/) crate. -The bindings reuse a **per-thread AST `Bump`** (`with_ast_arena`) that is `reset()` between calls rather than allocated fresh per call: the bindings are invoked once per file in tight loops, and per-call arena malloc/free churns the system allocator's heap high-water in a way that is measurable through a host FFI layer. `reset()` retains the largest chunk and rewinds, so a warm thread does no per-call malloc/free. The per-file AST is fully consumed before the next call's `reset()`, so the reuse is sound (incl. after a `catch_unwind`-caught panic). The `format` path additionally reuses a **per-thread doc arena** (`with_doc_arena`, the same shape over `DocArena` and calling each language's `format_in`). Both helpers live in the shared [`tsv_arena`](../tsv_arena/) crate (one copy for all three bindings — `tsv_ffi`, `tsv_napi`, `tsv_wasm`); this crate's `format` feature maps to `tsv_arena/format`, which pulls `tsv_lang` for the `DocArena` type, so the parse-only build stays lean. +The bindings reuse a **per-thread AST `Bump`** (`with_ast_arena`) that is `reset()` between calls rather than allocated fresh per call: the bindings are invoked once per file in tight loops, and per-call arena malloc/free churns the system allocator's heap high-water in a way that is measurable through a host FFI layer. `reset()` retains the largest chunk and rewinds, so a warm thread does no per-call malloc/free. The per-file AST is fully consumed before the next call's `reset()`, so the reuse is sound (incl. after a `catch_unwind`-caught panic). The `format` path additionally reuses a **per-thread doc arena** (`with_doc_arena`, the same shape over `DocArena` and calling each language's `format_folded_in` — the fold ahead of the parse hands its line verdict to the printer, so a document is walked once). Both helpers live in the shared [`tsv_arena`](../tsv_arena/) crate (one copy for all three bindings — `tsv_ffi`, `tsv_napi`, `tsv_wasm`); this crate's `format` feature maps to `tsv_arena/format`, which pulls `tsv_lang` for the `DocArena` type, so the parse-only build stays lean. Build/usage commands live in [../../CLAUDE.md §JS Bindings](../../CLAUDE.md#js-bindings). diff --git a/crates/tsv_ffi/src/lib.rs b/crates/tsv_ffi/src/lib.rs index d29fa6fc0..ead095b10 100644 --- a/crates/tsv_ffi/src/lib.rs +++ b/crates/tsv_ffi/src/lib.rs @@ -268,13 +268,13 @@ macro_rules! parse_format { // The format path's line-terminator fold, ahead of the parse — see // `tsv_lang::printing::normalize_carriage_returns`. `parse_convert!` deliberately // skips it: the wire's offsets are a drop-in contract over the author's own bytes. - let normalized = tsv_lang::printing::normalize_carriage_returns($source); - let source = normalized.as_ref(); + let folded = tsv_lang::printing::normalize_carriage_returns($source); + let source = folded.text(); with_ast_arena(|arena| { let ast = parse_ast!($goalness, $lang, source, $goal, arena).map_err(|e| e.to_string())?; Ok(with_doc_arena(|doc_arena| { - $lang::format_in(&ast, source, doc_arena) + $lang::format_folded_in(&ast, &folded, doc_arena) })) }) }}; diff --git a/crates/tsv_lang/CLAUDE.md b/crates/tsv_lang/CLAUDE.md index 29f28a34b..a9830b47d 100644 --- a/crates/tsv_lang/CLAUDE.md +++ b/crates/tsv_lang/CLAUDE.md @@ -16,7 +16,7 @@ Each module's visibility (in parens) reflects `pub use`-only modules (private) v - `comment` (`comment.rs`, private) — Comment type, classification, and O(log n) range lookup - `acorn_prefix` (`acorn_prefix.rs`, private) — `AcornPrefix` / `AcornPrefixText`: what acorn SAW in the text ahead of one embedded Svelte parse. Svelte hands acorn a differently prepared string at every island, and for four of its readers that string is **manufactured** — the prefix blanked (`replace(/[^\n]/g, ' ')`), or blanked and then capped with a synthetic token (`read_pattern`'s `(pattern = 1)`, `read_type_annotation`'s `_ as `), or blanked only over the non-whitespace (the `{#snippet}` head). Two wire answers read that preparation rather than the document, which is why one value states it: the **line class** an acorn-owned `loc` was counted under (`counts_ecmascript_lines`, the axis `tsv_ts::AcornSeed` seeds from) and the **indentation** `onComment` dedents a multi-line block comment by (`line_start` + `line_indentation`, read by `printing::strip_comment_indentation`). It lives here rather than beside the seed in `tsv_ts` because the dedent — `onComment`'s mirror — already does, and a fact two crates read is this crate's. ⚠️ The two synthetic tokens are not the same shape: `(` is **spliced** between the prefix and the region, where `_ as ` **overwrites** the five document bytes it covers — so it can swallow an author's `\n` (the line then opens further back than the document's does) and it ends the run at a byte the document has no `[ \t]` reading for. `tsv_svelte`'s parser records the preparation per region (`Root::acorn_regions`) and its writer looks it up by position, exactly as it does the seed. No fixture can carry most of this — the pins are `tests/comment_dedent_manufactured_source.rs` (each reader, each with its null control) and the unit tests beside the code; the template readers also ride the ``-frozen fixture `tests/fixtures/svelte/syntax/comments/head_multiline_comment_dedent` - `comment_ledger` (`comment_ledger.rs`, pub, **`comment_check` feature**) — the print-once comment ledger (diagnostic) -- `printing` (`printing.rs`, pub) — String literal formatting, same-line detection, visual width. The three line questions (`is_same_line_scan` / `has_newline_between_scan` / `has_blank_line_between_scan`) read the source bytes — a bounded scan past which the document's line-break table answers, and that table is built ON DEMAND: `LineBreaks` takes the document's verdict up front (`line_terminators_are_lf_only`, one loose-needle pass — is every terminator a `\n`?) and fills its arena-parked table (one entry per terminator's LAST byte, `build_line_breaks_into`) only from the `#[cold]` fallbacks; `LineTable` is the `Copy` handle the printers carry (`LineTable::EMPTY` is the canonical reprint's ERASED layout table, a distinct state answered before a byte is read). The `*_fast` forms binary-search a built table and remain the fallback's search and the tests' oracle — no printer calls them +- `printing` (`printing.rs`, pub) — String literal formatting, same-line detection, visual width. The three line questions (`is_same_line_scan` / `has_newline_between_scan` / `has_blank_line_between_scan`) read the source bytes — a bounded scan past which the document's line-break table answers, and that table is built ON DEMAND: `LineBreaks` takes the document's verdict up front (`line_terminators_are_lf_only`, one loose-needle pass — is every terminator a `\n`?) and fills its arena-parked table (one entry per terminator's LAST byte, `build_line_breaks_into`) only from the `#[cold]` fallbacks; `LineTable` is the `Copy` handle the printers carry (`LineTable::EMPTY` is the canonical reprint's ERASED layout table, a distinct state answered before a byte is read). The `*_fast` forms binary-search a built table and remain the fallback's search and the tests' oracle — no printer calls them. The format path's `` fold (`normalize_carriage_returns`) returns a `FoldedSource` — the folded text WITH that verdict, taken by the fold's own pass (`classify_line_terminators`, the same loose-needle loop, recording where the first `\r` is and whether a U+2028 / U+2029 is anywhere) — and `LineBreaks::of_folded` builds on it, so a document that folds is walked once: each language crate's `format_folded_in` is the entry point that takes it (the CLI, the three bindings, every `format_str`); `format_in` classifies the source itself - `source_scan` (`source_scan.rs`, pub) — Trivia-aware source scanning: the `skip_trivia` cursor and its run-level companion `skip_trivia_run` (`skip_trivia` answers "does trivia START here", `skip_trivia_run` "where does the trivia END" — the question a caller sitting *between two tokens* has, since a gap can alternate whitespace, comment, whitespace, comment; it also owns the two obligations each hand-rolled copy of that loop had to remember, that `skip_trivia` must not be called at `end` and that the whitespace step must move by whole characters, and takes the caller's own language whitespace class rather than `char::is_whitespace`), plus the `find_char` / `find_keyword` / `rfind_keyword` delimiter/keyword finders (skipping JS/CSS comments + strings), the regex helpers — `OperandAnchor` + `skip_regex_literal` (the one piece of `/`-disambiguation `skip_trivia` deliberately leaves out, since it needs previous-token context; `OperandAnchor` is the seam that carries it, deriving the regex-vs-division anchor where a `/` asks for it instead of maintaining it on every scanned byte, and the rule it owns — a string or template ends an operand, a comment does not — is stated only there), the **hop-needle contract** every scan that skips bytes instead of asking `skip_trivia` about each one must satisfy (`TRIVIA_OPENERS` + the `const`-evaluable `covers_trivia_openers`, paired in a `const _` beside each needle array so a hop that could step over a string or comment is a compile error; `trivia_hop_needles` builds the array for a scan whose own byte is only known at runtime, and `is_hop_needle` is the one-byte pre-test that pays in front of a hop whose runs are routinely empty — the choice between those rungs is a pure performance one that no gate can check, so the reason is written at each site), and the balanced-brace pair `scan_to_matching_brace` (the expression-context `{…}` matcher — trivia + regex + template aware) / `skip_template_literal` (interpolation-aware template skip, since `skip_trivia`'s opaque quote-to-quote scan mis-pairs backticks across a nested template like `` `${`x`}` ``). The single chokepoint for re-scanning source between AST nodes — used by AST conversion, all three printers, the Svelte parser (which wraps `scan_to_matching_brace` for its `{…}` tags and shares `skip_template_literal` in its regex-unaware binding-pattern scan), and the TS parser's arrow-vs-paren / type-args lookahead - `escapes` (`escapes.rs`, private) — Escape sequence handling (quote swapping) — used internally by `printing` - `whitespace` (`whitespace.rs`, private) — `is_js_whitespace`, the ECMAScript `\s` CharSet, shared because two language crates need it and neither can reach the other (`tsv_svelte`'s `is_svelte_ws` is this set; so is the class `parseCss` skips, and `tsv_css` is a *dependency* of `tsv_svelte`). One definition, one exhaustive per-code-point test against Svelte's own hand-written enumeration. ⚠️ Its module doc is the **workspace-wide whitespace index** — every class in tsv, which oracle each answers to, the reads that deliberately answer to none, and the four crates the `char::is_whitespace` / `str::trim` trap has been found in. Read it before adding or changing any whitespace predicate, in any crate: the recurring bug is a site's crate being taken as evidence about which class it wants @@ -34,7 +34,7 @@ The doc builder is the core of the formatting architecture. Language printers bu ### Key Types -- **`DocArena`** — Contiguous storage for all doc nodes, plus the text pool (the `String` backing `Pooled`/`MultilineText` bodies) and an inline direct-mapped static cache whose slots carry two halves: the amortized-eager widths behind `text()` statics, and the per-document **interned node** — repeated `text(",")` calls within one format return one shared `DocId` instead of allocating per call (`empty()` interns through a dedicated cell; sound because statics are position-free at render, nodes are append-only, and no consumer compares `DocId` identity). The stateless singleton nodes intern the same way through dedicated generation-gated cells with no hash probe: the four `Line` kinds (direct-indexed by `LineKind` discriminant), `LineSuffixBoundary`, `BreakParent`, and `FlushBreak` — a `Line` node carries no mode or indent (both supplied per visit by the enclosing render command), so every `line()`/`softline()`/`hardline()`/`literalline()` within one document returns one shared node. The arena also parks a per-render output scratch buffer (`take_render_scratch()`/`park_render_scratch()` — the render analog of `pool_writer()`'s parked scratch): the hot per-piece render-and-write seams (TS whole-program/per-expression, CSS per declaration, Svelte per root node) render through the `*_into` entry points into it, one warm buffer per file instead of an alloc/free per call, with a fresh-fallback empty default so nested renders stay correct. The render loop's work buffers pool the same way — each top-level render borrows the arena's command stack + line-suffix buffer (`RefCell`-backed, cleared at borrow; sub-renders keep their own inline `SmallVec` locals) — and the per-file line-break table parks via `take_line_breaks_scratch()`/`park_line_breaks_scratch()` (filled by `printing::build_line_breaks_into` in each `format_in`), and the multi-line block-comment builders borrow a parked line-offset scratch (`borrow_line_spans_scratch()` — one `printing::next_lf` pass per comment fills each body line's `(start, end)` range, so the classifier and builders iterate slice-cheap with no per-comment line buffer, and no line is materialized as a `str` to fill it). The doc-build side pools too: the wide-list builders assemble their parts into a `DocBuf` drawn from a recursion-safe free-list (`acquire_docbuf`/`release_docbuf`, or the `PooledDocBuf` RAII guard from `pooled_docbuf()`) — a builder pops a cleared buffer (retaining a prior spill's heap capacity) and returns it on scope exit, so the many transient `SmallVec` spills across a document collapse into a handful of long-lived reused buffers; the free-list keeps **only spilled buffers** (a release drops a never-spilled one — nothing to retain, free to re-construct), so every pooled entry carries real heap capacity and a big-need builder can't pop a virgin buffer while capacity sits deeper in the LIFO; retained across `reset()`; byte-identical — allocation only, never output. A parked node-keyed doc-share map (`share_map_scratch()`, an AST-node pointer → built `DocId` table) backs the TS printer's member-chain argument sharing the same way — the consumer clears it at share-scope entry/exit, so only its table capacity persists instead of a fresh `HashMap` resize chain per printer/file. Heuristic capacity: ~2 nodes per source byte (kept above the post-interning ~0.26/byte density because `estimated_children = nodes/2` must still clear the un-shrunk children demand); the text pool pre-sizes at source/8 (measured per-file demand p50 ≈ 0.17× source). `reset()` clears the node/child/text-pool/memo stores while retaining capacity — O(1) on the node store, since `DocNode` carries no drop glue — so a multi-file driver reuses one arena across files (the doc-IR analogue of the binding crates' `Bump::reset()` reuse); the static cache's width halves deliberately survive `reset()` (they key on `'static` string addresses — warming once per arena lifetime) while the interned node halves are invalidated in O(1) by the reset's `format_gen` bump; the printers borrow `&DocArena` and the caller owns the reusable one (`format_in` on each language crate is the borrowed-arena entry point). +- **`DocArena`** — Contiguous storage for all doc nodes, plus the text pool (the `String` backing `Pooled`/`MultilineText` bodies) and an inline direct-mapped static cache whose slots carry two halves: the amortized-eager widths behind `text()` statics, and the per-document **interned node** — repeated `text(",")` calls within one format return one shared `DocId` instead of allocating per call (`empty()` interns through a dedicated cell; sound because statics are position-free at render, nodes are append-only, and no consumer compares `DocId` identity). The stateless singleton nodes intern the same way through dedicated generation-gated cells with no hash probe: the four `Line` kinds (direct-indexed by `LineKind` discriminant), `LineSuffixBoundary`, `BreakParent`, and `FlushBreak` — a `Line` node carries no mode or indent (both supplied per visit by the enclosing render command), so every `line()`/`softline()`/`hardline()`/`literalline()` within one document returns one shared node. The arena also parks a per-render output scratch buffer (`take_render_scratch()`/`park_render_scratch()` — the render analog of `pool_writer()`'s parked scratch): the hot per-piece render-and-write seams (TS whole-program/per-expression, CSS per declaration, Svelte per root node) render through the `*_into` entry points into it, one warm buffer per file instead of an alloc/free per call, with a fresh-fallback empty default so nested renders stay correct. The render loop's work buffers pool the same way — each top-level render borrows the arena's command stack + line-suffix buffer (`RefCell`-backed, cleared at borrow; sub-renders keep their own inline `SmallVec` locals) — and the per-file line-break table parks via `take_line_breaks_scratch()`/`park_line_breaks_scratch()` (a `printing::LineBreaks` per `format_in`, filled on demand from the line questions' cold fallbacks), and the multi-line block-comment builders borrow a parked line-offset scratch (`borrow_line_spans_scratch()` — one `printing::next_lf` pass per comment fills each body line's `(start, end)` range, so the classifier and builders iterate slice-cheap with no per-comment line buffer, and no line is materialized as a `str` to fill it). The doc-build side pools too: the wide-list builders assemble their parts into a `DocBuf` drawn from a recursion-safe free-list (`acquire_docbuf`/`release_docbuf`, or the `PooledDocBuf` RAII guard from `pooled_docbuf()`) — a builder pops a cleared buffer (retaining a prior spill's heap capacity) and returns it on scope exit, so the many transient `SmallVec` spills across a document collapse into a handful of long-lived reused buffers; the free-list keeps **only spilled buffers** (a release drops a never-spilled one — nothing to retain, free to re-construct), so every pooled entry carries real heap capacity and a big-need builder can't pop a virgin buffer while capacity sits deeper in the LIFO; retained across `reset()`; byte-identical — allocation only, never output. A parked node-keyed doc-share map (`share_map_scratch()`, an AST-node pointer → built `DocId` table) backs the TS printer's member-chain argument sharing the same way — the consumer clears it at share-scope entry/exit, so only its table capacity persists instead of a fresh `HashMap` resize chain per printer/file. Heuristic capacity: ~2 nodes per source byte (kept above the post-interning ~0.26/byte density because `estimated_children = nodes/2` must still clear the un-shrunk children demand); the text pool pre-sizes at source/8 (measured per-file demand p50 ≈ 0.17× source). `reset()` clears the node/child/text-pool/memo stores while retaining capacity — O(1) on the node store, since `DocNode` carries no drop glue — so a multi-file driver reuses one arena across files (the doc-IR analogue of the binding crates' `Bump::reset()` reuse); the static cache's width halves deliberately survive `reset()` (they key on `'static` string addresses — warming once per arena lifetime) while the interned node halves are invalidated in O(1) by the reset's `format_gen` bump; the printers borrow `&DocArena` and the caller owns the reusable one (`format_in` on each language crate is the borrowed-arena entry point). - **`DocId`** (`u32`) — Lightweight, `Copy` handle into the arena. No cloning, no recursive Drop. - **`DocBuf`** (`SmallVec<[DocId; 8]>`) — Shared stack buffer for assembling a node's doc parts before `concat()` / `fill()`. Most nodes have only a handful of parts, so the common case stays off the heap; larger nodes spill. Used by all language printers (the TS chain / binary-operator printers, the Svelte template printer) as the single canonical doc-parts buffer type. Wide-list builders (statement / object / array / parameter / specifier lists) draw a reusable buffer from the arena's `DocBuf` free-list (`pooled_docbuf()`) rather than allocating a fresh `SmallVec` per call, amortizing the per-spill malloc/free churn (see `DocArena` below). - **`DocNode`** — Node variants: `Text`, `MultilineText` (a `\n`-separated body rendered with per-line context indent — one pool-stored body for an indentable multi-line block comment), `Line`, `Indent`, `Dedent`, `Group`, `IfBreak`, `Concat`, `Fill`, etc. `DocNode` carries no drop glue (`const`-asserted via `needs_drop`): dynamic text lives in the arena text pool, so `reset()`/drop never walk the node store running destructors. Its size is also pinned by a companion `const` assert — **24 B on 64-bit** (the native flagship), **16 B on wasm32** (the shipped WASM bundles); the size is pointer-width dependent (`AlignRoot`'s `usize`, `DocText::Static`'s fat pointer), so the pin is `cfg`-gated per target. The node store is walked linearly at render, so the AoS layout's cache locality is the point (shrinking the node has been refuted repeatedly on this traversal-bound engine); a variant that bloats it is a deliberate decision, not an accident. ⚠️ The 24 B comes from niche-packing into `Text`'s `DocText` payload, and that packing is **charged back at every `match` over a `DocNode`**: `DocText`'s four sub-tags own discriminant values 0..=3, so a kind switch must fold them together before it can index its jump table. The render loop and the fits walk both peel the fold off by probing ahead of the dispatch (the `Text` test *is* the fold), and in the fits walk the same probe also retires the memo round-trip for a leaf text; the same peel is a measured **regression** in `subtree_layout_fill`, whose commonest kind sits above the fold — it pays only where the peeled kind IS the fold's range. See the comments at all three sites and [architecture.md §DocText](../../docs/architecture.md#doctext-static-pooled-sourcespan-verbatimspan). diff --git a/crates/tsv_lang/src/printing.rs b/crates/tsv_lang/src/printing.rs index 70022cfae..b927034b4 100644 --- a/crates/tsv_lang/src/printing.rs +++ b/crates/tsv_lang/src/printing.rs @@ -354,10 +354,27 @@ pub fn ecmascript_lines(text: &str) -> impl Iterator { /// (`` → U+2028), so folding one would change what a template renders. /// /// Idempotent: its own output holds no `` to fold. +/// +/// **The fold's pass also takes the folded document's line verdict** — whether every +/// terminator left in it is a `\n` ([`FoldedSource::lf_only`]) — because the one loose +/// needle that finds a `\r` (`\r` or any non-ASCII byte, [`classify_line_terminators`]) +/// is the needle the verdict pass asks too ([`line_terminators_are_lf_only`]), and a +/// printer built on the fold ([`LineBreaks::of_folded`]) would otherwise walk every byte a +/// second time to re-ask it. The verdict is a fact about the FOLDED text: no `\r` remains +/// in it, so it is exactly "no U+2028 / U+2029 anywhere", and the fold moves neither. #[must_use] -pub fn normalize_carriage_returns(source: &str) -> Cow<'_, str> { - let Some(first) = source.find('\r') else { - return Cow::Borrowed(source); +pub fn normalize_carriage_returns(source: &str) -> FoldedSource<'_> { + let Terminators { + first_cr, + holds_separator, + } = classify_line_terminators(source.as_bytes()); + let lf_only = !holds_separator; + let Some(first) = first_cr else { + debug_assert_eq!(lf_only, line_terminators_are_lf_only(source.as_bytes())); + return FoldedSource { + text: Cow::Borrowed(source), + lf_only, + }; }; let mut out = String::with_capacity(source.len()); out.push_str(&source[..first]); @@ -371,7 +388,41 @@ pub fn normalize_carriage_returns(source: &str) -> Cow<'_, str> { rest = after.strip_prefix('\n').unwrap_or(after); } out.push_str(rest); - Cow::Owned(out) + debug_assert_eq!(lf_only, line_terminators_are_lf_only(out.as_bytes())); + FoldedSource { + text: Cow::Owned(out), + lf_only, + } +} + +/// A document with its `` fold applied ([`normalize_carriage_returns`]) and the line +/// verdict the fold's own pass took over the folded text — so a printer built on it +/// ([`LineBreaks::of_folded`]) takes the verdict from the pass that already ran instead +/// of walking the source again. The two travel as one value so the verdict can never be +/// read against a text it was not taken on. +#[derive(Debug)] +pub struct FoldedSource<'a> { + text: Cow<'a, str>, + lf_only: bool, +} + +impl<'a> FoldedSource<'a> { + /// The folded text — borrowed when the source held no ``. + pub fn text(&self) -> &str { + &self.text + } + + /// Whether every line terminator in [`Self::text`] is a `\n` — the fact + /// [`line_terminators_are_lf_only`] states over those bytes, here taken by the fold's + /// pass (a `\r` cannot remain, so this is exactly "no U+2028 / U+2029 anywhere"). + pub fn lf_only(&self) -> bool { + self.lf_only + } + + /// The folded text alone, for a caller with no printer to hand the verdict to. + pub fn into_text(self) -> Cow<'a, str> { + self.text + } } /// The line `position` sits on, as `(line_start, line_end, line_number)` — bounds in bytes, @@ -785,6 +836,22 @@ impl<'s> LineBreaks<'s> { Self::new(source, Vec::new()) } + /// [`Self::new`] over a folded document, taking the verdict the fold's own pass + /// already took ([`FoldedSource::lf_only`]) instead of classifying the bytes again — + /// the format entry points that fold ahead of the parse (the CLI, the bindings, each + /// crate's `format_str`) build their table this way, so the document is walked once, + /// not twice. + pub fn of_folded(folded: &'s FoldedSource<'_>, scratch: Vec) -> Self { + let bytes = folded.text().as_bytes(); + debug_assert_eq!(folded.lf_only(), line_terminators_are_lf_only(bytes)); + LineBreaks { + source: bytes, + lf_only: folded.lf_only(), + table: OnceCell::new(), + scratch: Cell::new(scratch), + } + } + /// Whether every line terminator in the document is a `\n` (a `\r\n` counts: the /// byte the table records for it IS the `\n`; a bare `\r` or a U+2028 / U+2029 does /// not) — the document's verdict, taken once at construction. @@ -870,8 +937,9 @@ impl LineTable<'_> { // The document's verdict — one pass, ahead of any table // -/// [`line_terminators_are_lf_only`]'s loose lane test — `\r` or any non-ASCII byte — is a -/// superset of the exact candidate class `{ \r, 0xE2 }` its word re-ask and tail answer, +/// [`line_terminators_are_lf_only`]'s loose lane test — `\r` or any non-ASCII byte, and +/// [`classify_line_terminators`]'s, which walks the same loop — is a superset of the exact +/// candidate class `{ \r, 0xE2 }` their word re-asks and tails answer, /// proved here rather than trusted (a uniform word cannot borrow across lanes unless it is /// itself a match, so `hits != 0` is exactly the byte test). const _: () = { @@ -891,7 +959,10 @@ const _: () = { /// table records for it is the `\n`); a bare `\r` or a U+2028 / U+2029 anywhere, a string /// literal or a comment body included, says no. The document-level fact the scan forms of /// the three line questions are gated on, and the one thing a document pays for up front -/// now that its table is built on demand ([`LineBreaks`]). +/// now that its table is built on demand ([`LineBreaks`]) — on the entry points that fold +/// `` ahead of the parse, the fold's own pass takes it instead +/// ([`classify_line_terminators`], the same loop with a different cold re-ask), so this +/// runs only for a caller that never folds (`format_in` reached directly). /// /// One pass with ONE loose needle: `\r` or any non-ASCII byte, three operations a word /// ([`crate::swar::zero_or_high_lanes`]), and nearly every word of real source fires @@ -980,6 +1051,107 @@ fn lf_only_tail(bytes: &[u8], from: usize) -> bool { .all(|i| !matches!(bytes[i], b'\r' | LINE_SEPARATOR_LEAD) || lf_only_at(bytes, i)) } +/// What the `` fold's one pass learns about a document ([`classify_line_terminators`]): +/// where its first `\r` is, if it holds one — where the fold starts copying — and whether +/// a U+2028 / U+2029 is anywhere in it — the folded text's line verdict, negated. +struct Terminators { + first_cr: Option, + holds_separator: bool, +} + +/// The `` fold's pass ([`normalize_carriage_returns`]): [`line_terminators_are_lf_only`]'s +/// loose loop — the same one needle, `\r` or any non-ASCII byte, two words a step, a +/// fired word re-asked out of line — recording the two facts the fold and the folded +/// document's verdict need, so the fold's up-front `find('\r')` (std's memchr, nine +/// instructions a word — a whole-source pass on every format entry point that folds, over +/// a corpus in which no file holds a `\r`) is gone and the verdict pass does not run a +/// second time behind it; the fold's per-line search from the first `\r` on is unchanged. Unlike the verdict pass it has +/// no early answer to return: it runs to the end, or until both facts are known. +/// +/// The verdict is stated over the FOLDED text, which is why a `\r` is not asked whether +/// it ends in a `\n` here: after the fold every `\r` and `\r\n` IS a `\n`, and a U+2028 / +/// U+2029 — which the fold does not touch — is the only terminator that can remain +/// otherwise. +/// +/// Outlined on purpose, like the verdict pass: inlined into the CLI's format function the +/// loop carried that function's register pressure — 19 instructions per sixteen bytes +/// against 17 here — and read 0.07 points less of a CLI run (measured on two corpora). +#[inline(never)] +fn classify_line_terminators(bytes: &[u8]) -> Terminators { + let mut found = Terminators { + first_cr: None, + holds_separator: false, + }; + let mut i = 0; + // The loop is `line_terminators_are_lf_only`'s, spelled the same way for the same + // reasons (two words a step; the sixteen bytes claimed once). + while let Some(chunk) = bytes[i..].first_chunk::<16>() { + let (words, _) = chunk.as_chunks::<8>(); + let (a, b) = (u64::from_le_bytes(words[0]), u64::from_le_bytes(words[1])); + if (zero_or_high_lanes(a ^ splat(b'\r')) | zero_or_high_lanes(b ^ splat(b'\r'))) != 0 + && (classify_word(bytes, i, a, &mut found) + || classify_word(bytes, i + 8, b, &mut found)) + { + return found; + } + i += 16; + } + // The one word that may remain ahead of the tail. + if let Some(chunk) = bytes[i..].first_chunk::<8>() { + let w = u64::from_le_bytes(*chunk); + if zero_or_high_lanes(w ^ splat(b'\r')) != 0 && classify_word(bytes, i, w, &mut found) { + return found; + } + i += 8; + } + for at in i..bytes.len() { + if matches!(bytes[at], b'\r' | LINE_SEPARATOR_LEAD) && classify_at(bytes, at, &mut found) { + break; + } + } + found +} + +/// [`classify_line_terminators`]'s exact question over the one word at `at` the loose +/// test fired on — every `\r` and every `0xE2` lane in it, asked [`classify_at`]. Out of +/// line for the reason [`lf_only_in_word`] is: the loose loop's unit holds one loop and +/// nothing else, and this runs on a fraction of a percent of the words. Returns whether +/// both facts are now known, so the pass can stop. +#[cold] +#[inline(never)] +fn classify_word(bytes: &[u8], at: usize, w: u64, found: &mut Terminators) -> bool { + let mut hits = zero_lanes(w ^ splat(b'\r')) | zero_lanes(w ^ splat(LINE_SEPARATOR_LEAD)); + while hits != 0 { + let lane = (hits.trailing_zeros() / 8) as usize; + if classify_at(bytes, at + lane, found) { + return true; + } + hits &= hits - 1; + } + false +} + +/// Record what the candidate byte at `at` is — the first `\r` seen, a U+2028 / U+2029, or +/// (a `0xE2` that leads another character, or a lane the SWAR kernel flagged spuriously) +/// nothing. Returns whether both facts are now known. +#[inline] +fn classify_at(bytes: &[u8], at: usize, found: &mut Terminators) -> bool { + match bytes[at] { + b'\r' => { + if found.first_cr.is_none() { + found.first_cr = Some(at); + } + } + LINE_SEPARATOR_LEAD => { + if line_terminator_len(bytes, at).is_some() { + found.holds_separator = true; + } + } + _ => {} + } + found.first_cr.is_some() && found.holds_separator +} + // // The same three questions answered by a bounded SCAN of the source, with the table // as the fallback @@ -2966,14 +3138,14 @@ mod tests { /// Every spelling of a carriage return folds to LF, and a CRLF pair stays ONE terminator. #[test] fn carriage_returns_normalize_to_lf() { - assert_eq!(normalize_carriage_returns("a\r\nb"), "a\nb"); - assert_eq!(normalize_carriage_returns("a\rb"), "a\nb"); - assert_eq!(normalize_carriage_returns("a\r\n\r\nb"), "a\n\nb"); - assert_eq!(normalize_carriage_returns("a\r\rb"), "a\n\nb"); + assert_eq!(normalize_carriage_returns("a\r\nb").text(), "a\nb"); + assert_eq!(normalize_carriage_returns("a\rb").text(), "a\nb"); + assert_eq!(normalize_carriage_returns("a\r\n\r\nb").text(), "a\n\nb"); + assert_eq!(normalize_carriage_returns("a\r\rb").text(), "a\n\nb"); // `\n\r` is two terminators, not a pair — only `\r\n` is one. - assert_eq!(normalize_carriage_returns("a\n\rb"), "a\n\nb"); - assert_eq!(normalize_carriage_returns("\r"), "\n"); - assert_eq!(normalize_carriage_returns("\r\n"), "\n"); + assert_eq!(normalize_carriage_returns("a\n\rb").text(), "a\n\nb"); + assert_eq!(normalize_carriage_returns("\r").text(), "\n"); + assert_eq!(normalize_carriage_returns("\r\n").text(), "\n"); } /// A CR-free string comes back BORROWED — the fold runs ahead of every format, so the @@ -2981,27 +3153,61 @@ mod tests { #[test] fn carriage_return_normalization_borrows_without_one_and_is_idempotent() { assert!(matches!( - normalize_carriage_returns("a\nb\n"), + normalize_carriage_returns("a\nb\n").into_text(), Cow::Borrowed("a\nb\n") )); - assert!(matches!(normalize_carriage_returns(""), Cow::Borrowed(""))); - let once = normalize_carriage_returns("a\r\nb\rc").into_owned(); assert!(matches!( - normalize_carriage_returns(&once), + normalize_carriage_returns("").into_text(), + Cow::Borrowed("") + )); + let once = normalize_carriage_returns("a\r\nb\rc") + .into_text() + .into_owned(); + assert!(matches!( + normalize_carriage_returns(&once).into_text(), Cow::Borrowed(_) )); assert_eq!(once, "a\nb\nc"); } + /// The verdict the fold's pass takes is the verdict over the FOLDED text: a `\r` or a + /// `\r\n` is a `\n` on the other side of the fold, so only a U+2028 / U+2029 can say + /// no — and the fold does not move those. (The exhaustive test below grades the same + /// claim at every alignment of every terminator shape; these are the shapes by name.) + #[test] + fn the_fold_takes_the_folded_texts_line_verdict() { + assert!(normalize_carriage_returns("a\nb").lf_only()); + assert!(normalize_carriage_returns("a\r\nb").lf_only()); + assert!(normalize_carriage_returns("a\rb").lf_only()); + assert!(normalize_carriage_returns("a\u{2000}b\u{e9}").lf_only()); + assert!(!normalize_carriage_returns("a\u{2028}b").lf_only()); + assert!(!normalize_carriage_returns("a\u{2029}b").lf_only()); + assert!(!normalize_carriage_returns("a\u{2028}\r\nb").lf_only()); + assert!(!normalize_carriage_returns("a\r\n\u{2028}b").lf_only()); + // Both facts known early: the pass may stop, and the first `\r` is still the FIRST. + let folded = normalize_carriage_returns("\r\u{2028}x\ry\r\nz"); + assert_eq!(folded.text(), "\n\u{2028}x\ny\nz"); + assert!(!folded.lf_only()); + } + /// U+2028 / U+2029 are terminators to ECMAScript and ordinary characters to HTML and CSS /// text. Both formatters keep them where the author put them, and ECMAScript's own TRV /// keeps each as itself, so this fold must not reach them even though /// `line_terminator_len` counts them. #[test] fn carriage_return_normalization_leaves_line_and_paragraph_separators_alone() { - assert_eq!(normalize_carriage_returns("a\u{2028}b"), "a\u{2028}b"); - assert_eq!(normalize_carriage_returns("a\u{2029}b"), "a\u{2029}b"); - assert_eq!(normalize_carriage_returns("a\u{2028}\r\nb"), "a\u{2028}\nb"); + assert_eq!( + normalize_carriage_returns("a\u{2028}b").text(), + "a\u{2028}b" + ); + assert_eq!( + normalize_carriage_returns("a\u{2029}b").text(), + "a\u{2029}b" + ); + assert_eq!( + normalize_carriage_returns("a\u{2028}\r\nb").text(), + "a\u{2028}\nb" + ); } #[test] @@ -3161,6 +3367,22 @@ mod tests { lf_only, "verdict with a non-ASCII byte ahead {ahead:?}" ); + // The fold's own pass states the folded text's verdict — the up-front + // verdict over the bytes it returns — at the same alignments. + for document in [&padded, &ahead] { + let folded = normalize_carriage_returns(document); + assert_eq!( + folded.lf_only(), + line_terminators_are_lf_only(folded.text().as_bytes()), + "fold verdict {document:?}" + ); + assert!(!folded.text().contains('\r'), "fold left a CR {document:?}"); + assert_eq!( + matches!(folded.into_text(), Cow::Borrowed(_)), + !document.contains('\r'), + "fold borrowed/copied wrongly {document:?}" + ); + } } if lf_only { lf_only_documents += 1; diff --git a/crates/tsv_napi/CLAUDE.md b/crates/tsv_napi/CLAUDE.md index e5d41d753..a53dece12 100644 --- a/crates/tsv_napi/CLAUDE.md +++ b/crates/tsv_napi/CLAUDE.md @@ -8,7 +8,7 @@ Depends on `tsv_ts`, `tsv_css`, `tsv_svelte`. The **Node/Bun** sibling of the bi This is a **tsv-scoped carve-out** from the ecosystem N-API deferral — **not** an ecosystem-wide flip. -Like `tsv_ffi`, the bindings reuse a **per-thread AST `Bump`** (`with_ast_arena`) that is `reset()` between calls rather than allocated fresh per call — the bindings are invoked once per file in tight loops, and per-call arena malloc/free churns the system allocator's heap high-water in a way that is measurable through a binding layer. The `format` path likewise reuses a **per-thread doc arena** (`with_doc_arena`, the same shape over `DocArena`, calling each language's `format_in`). Both helpers live in the shared [`tsv_arena`](../tsv_arena/) crate (used by all three bindings — `tsv_ffi`, `tsv_napi`, and `tsv_wasm` — so there's one copy of the subtle reuse/soundness contract, not three hand-synced ones). This crate's `format` feature maps to `tsv_arena/format`, which pulls `tsv_lang` for the `DocArena` type; the parse-only build leaves it off and stays lean. +Like `tsv_ffi`, the bindings reuse a **per-thread AST `Bump`** (`with_ast_arena`) that is `reset()` between calls rather than allocated fresh per call — the bindings are invoked once per file in tight loops, and per-call arena malloc/free churns the system allocator's heap high-water in a way that is measurable through a binding layer. The `format` path likewise reuses a **per-thread doc arena** (`with_doc_arena`, the same shape over `DocArena`, calling each language's `format_folded_in` — the fold ahead of the parse hands its line verdict to the printer, so a document is walked once). Both helpers live in the shared [`tsv_arena`](../tsv_arena/) crate (used by all three bindings — `tsv_ffi`, `tsv_napi`, and `tsv_wasm` — so there's one copy of the subtle reuse/soundness contract, not three hand-synced ones). This crate's `format` feature maps to `tsv_arena/format`, which pulls `tsv_lang` for the `DocArena` type; the parse-only build leaves it off and stays lean. Build/usage commands live in [../../CLAUDE.md §JS Bindings](../../CLAUDE.md#js-bindings). diff --git a/crates/tsv_napi/src/lib.rs b/crates/tsv_napi/src/lib.rs index 6532cd62c..6f24d8e60 100644 --- a/crates/tsv_napi/src/lib.rs +++ b/crates/tsv_napi/src/lib.rs @@ -97,13 +97,13 @@ macro_rules! parse_format { // The format path's line-terminator fold, ahead of the parse — see // `tsv_lang::printing::normalize_carriage_returns`. `parse_convert!` deliberately // skips it: the wire's offsets are a drop-in contract over the author's own bytes. - let normalized = tsv_lang::printing::normalize_carriage_returns($source); - let source = normalized.as_ref(); + let folded = tsv_lang::printing::normalize_carriage_returns($source); + let source = folded.text(); with_ast_arena(|arena| { let ast = parse_ast!($goalness, $lang, source, $goal, arena) .map_err(|e| napi::Error::from_reason(e.to_string()))?; Ok(with_doc_arena(|doc_arena| { - $lang::format_in(&ast, source, doc_arena) + $lang::format_folded_in(&ast, &folded, doc_arena) })) }) }}; diff --git a/crates/tsv_svelte/CLAUDE.md b/crates/tsv_svelte/CLAUDE.md index 2bdecd7af..a32942d40 100644 --- a/crates/tsv_svelte/CLAUDE.md +++ b/crates/tsv_svelte/CLAUDE.md @@ -20,8 +20,8 @@ See [../../CLAUDE.md §Project Structure](../../CLAUDE.md#project-structure) for `src/lib.rs` exports free functions matching the tsv pattern: - `parse(source, arena: &'arena Bump) -> Result>` — internal AST, bump-arena-allocated (caller owns the `Bump`). The Svelte parser creates the one arena per document and shares it with every embedded sub-AST — `tsv_ts` (`