diff --git a/docs/developers-guide.md b/docs/developers-guide.md index 275c7a46..5759ea93 100644 --- a/docs/developers-guide.md +++ b/docs/developers-guide.md @@ -648,6 +648,20 @@ Together, these helpers make the rest of the processing pipeline deal with a single normalized fence, and avoid carrying separate logic for detached specifier lines. +### Strategy enum + +`Strategy` is the canonical private choice for fence-marker rewriting. Its +variants have the following effects: + +- `Compress` rewrites the marker to exactly three backticks while preserving + indentation and the language specifier. +- `Preserve` retains the original marker character and run length when an + interior fence would otherwise become structural. + +All fence-marker rewriting must dispatch on `Strategy` through `rewrite_marker`. +`flush_matched_block` selects the matched-block strategy, while +`rewrite_fence_line` only dispatches it and falls back to the original line. + ### Architecture The earlier implementation used two slice-and-index helpers, diff --git a/src/fences.rs b/src/fences.rs index 7b15a569..ac50eb3c 100644 --- a/src/fences.rs +++ b/src/fences.rs @@ -17,6 +17,10 @@ use crate::wrap::{FenceObservation, FenceTracker, ObservedFence}; mod attachment; +#[cfg(test)] +#[path = "fences_properties.rs"] +mod properties; + use attachment::attach_to_next_fence; static FENCE_RE: LazyLock = lazy_regex!( @@ -75,24 +79,18 @@ fn normalize_specifier(line: &str) -> (String, String) { (cleaned, indent) } +/// Select how a recognized fence marker is normalized. #[derive(Clone, Copy)] -enum FenceRewrite { +enum Strategy { Compress, - PreserveDelimiters, -} - -#[derive(Clone, Copy)] -enum MarkerStrategy { - Compressed, - PreserveDelimiter, + Preserve, } /// A retained source line together with its compressed rewrite, computed once /// when the line was parsed. /// -/// Caching `compressed` here lets every flush path emit the line without running -/// the normalization regex again, including `flush_unmatched_block`, which may -/// rewrite any retained line. +/// Caching `compressed` avoids repeated compression work and supports +/// `flush_unmatched_block`, which rewrites only the opening delimiter. struct CachedLine { line: String, /// The line rewritten with a compressed three-backtick delimiter, or `None` @@ -107,14 +105,14 @@ struct PendingFenceBlock { fn marker_char(marker: &str) -> Option { marker.chars().next() } -fn rewrite_marker(line: &str, strategy: MarkerStrategy) -> Option { +fn rewrite_marker(line: &str, strategy: Strategy) -> Option { let cap = FENCE_RE.captures(line)?; let indent = cap.get(1).map_or("", |m| m.as_str()); let original_marker = cap.get(2).map_or("", |m| m.as_str()); let lang = cap.get(3).map_or("", |m| m.as_str()); let marker = match strategy { - MarkerStrategy::Compressed => "```", - MarkerStrategy::PreserveDelimiter => original_marker, + Strategy::Compress => "```", + Strategy::Preserve => original_marker, }; Some(if is_null_lang(lang) { format!("{indent}{marker}") @@ -123,14 +121,6 @@ fn rewrite_marker(line: &str, strategy: MarkerStrategy) -> Option { }) } -fn compressed_fence_line(line: &str) -> Option { - rewrite_marker(line, MarkerStrategy::Compressed) -} - -fn preserved_fence_line(line: &str) -> Option { - rewrite_marker(line, MarkerStrategy::PreserveDelimiter) -} - fn interior_fence_requires_preserved_delimiters( opening_marker: &str, parsed: Option<(&str, &str, &str)>, @@ -147,24 +137,24 @@ fn interior_fence_requires_preserved_delimiters( marker_ch == opening_ch || marker_ch == '`' } -fn opening_rewrite(has_conflicting_interior_fence: bool) -> FenceRewrite { +fn opening_rewrite(has_conflicting_interior_fence: bool) -> Strategy { if has_conflicting_interior_fence { - FenceRewrite::PreserveDelimiters + Strategy::Preserve } else { - FenceRewrite::Compress + Strategy::Compress } } -/// Emit a delimiter line, reusing its cached compressed rewrite for the +/// Emit a fence line, reusing its cached compressed rewrite for the /// `Compress` strategy and computing the preserved rewrite on demand. /// -/// The `PreserveDelimiters` strategy is only chosen once per block, so its +/// The `Preserve` strategy is only chosen once per block, so its /// rewrite is not worth caching per line. -fn rewrite_delimiter(cached: CachedLine, rewrite: FenceRewrite) -> String { +fn rewrite_fence_line(cached: CachedLine, strategy: Strategy) -> String { let CachedLine { line, compressed } = cached; - match rewrite { - FenceRewrite::Compress => compressed.unwrap_or(line), - FenceRewrite::PreserveDelimiters => preserved_fence_line(&line).unwrap_or(line), + match strategy { + Strategy::Compress => compressed.unwrap_or(line), + Strategy::Preserve => rewrite_marker(&line, Strategy::Preserve).unwrap_or(line), } } fn flush_unmatched_block(block: PendingFenceBlock, out: &mut Vec) { @@ -186,7 +176,7 @@ fn flush_matched_block(block: PendingFenceBlock, out: &mut Vec) { let closing_index = block.lines.len() - 1; for (index, cached) in block.lines.into_iter().enumerate() { let emitted = if index == 0 || index == closing_index { - rewrite_delimiter(cached, rewrite) + rewrite_fence_line(cached, rewrite) } else { cached.line }; @@ -235,7 +225,7 @@ impl<'a> ParsedLine<'a> { line, observation: observed.observation, fence: observed.fence, - compressed: compressed_fence_line(line), + compressed: rewrite_marker(line, Strategy::Compress), } } diff --git a/src/fences_properties.rs b/src/fences_properties.rs new file mode 100644 index 00000000..f2cf7067 --- /dev/null +++ b/src/fences_properties.rs @@ -0,0 +1,79 @@ +//! Test-only property-test companion to the `fences` module. +//! +//! These tests generate syntactically valid fence lines to verify the private +//! `Strategy` dispatch contract independently of matched-block tracking. +//! Kani is deliberately not used: the project has no Kani dev-dependency or +//! harness infrastructure, while these generated cases cover both finite +//! strategy states across the bounded marker lengths relevant to this module. + +use proptest::{prelude::*, strategy::Strategy as ProptestStrategy}; + +use super::{Strategy, rewrite_marker}; + +fn fence_line_strategy() -> impl ProptestStrategy { + ( + prop::collection::vec(Just(' '), 0..=8), + prop_oneof![Just('`'), Just('~')], + 3_usize..=10, + prop_oneof![ + Just(String::new()), + Just("null".to_owned()), + Just("NULL".to_owned()), + Just("Null".to_owned()), + "[a-z][a-z0-9_+.-]{0,8}".prop_filter("excludes null language variants", |language| { + !language.eq_ignore_ascii_case("null") + }), + ], + ) + .prop_map(|(indent, marker, marker_length, language)| { + let indent: String = indent.into_iter().collect(); + let marker_run: String = std::iter::repeat_n(marker, marker_length).collect(); + let line = format!("{indent}{marker_run}{language}"); + (line, indent, marker, marker_length, language) + }) +} + +proptest! { + #[test] + fn compress_writes_three_backticks_and_preserves_indent_and_language( + (line, indent, _marker, _marker_length, language) in fence_line_strategy(), + ) { + let rewritten = rewrite_marker(&line, Strategy::Compress).expect("generated fence matches"); + + let expected_language = if language.eq_ignore_ascii_case("null") { + "" + } else { + &language + }; + + prop_assert_eq!(rewritten, format!("{indent}```{expected_language}")); + } + + #[test] + fn preserve_retains_the_original_marker_run( + (line, indent, marker, marker_length, language) in fence_line_strategy(), + ) { + let rewritten = rewrite_marker(&line, Strategy::Preserve).expect("generated fence matches"); + let expected_marker: String = std::iter::repeat_n(marker, marker_length).collect(); + + let expected_language = if language.eq_ignore_ascii_case("null") { + "" + } else { + &language + }; + + prop_assert_eq!(rewritten, format!("{indent}{expected_marker}{expected_language}")); + } + + #[test] + fn rewriting_is_idempotent( + (line, _indent, _marker, _marker_length, _language) in fence_line_strategy(), + ) { + for strategy in [Strategy::Compress, Strategy::Preserve] { + let once = rewrite_marker(&line, strategy).expect("generated fence matches"); + let twice = rewrite_marker(&once, strategy).expect("rewritten fence matches"); + + prop_assert_eq!(twice, once); + } + } +} diff --git a/tests/fences.rs b/tests/fences.rs index 0434c2eb..222a6373 100644 --- a/tests/fences.rs +++ b/tests/fences.rs @@ -3,9 +3,35 @@ #[macro_use] #[path = "common/mod.rs"] mod common; + use mdtablefix::{attach_orphan_specifiers, compress_fences}; use rstest::rstest; +fn assert_fence_snapshot(name: &str, input: &[String]) { + insta::with_settings!({ + snapshot_path => "snapshots", + prepend_module_to_snapshot => false, + }, { + insta::assert_snapshot!(name, compress_fences(input).join("\n")); + }); +} + +#[rstest] +#[case( + "fences_compress_overlong_backticks", + lines_vec!["`````rust", "fn main() {}", "`````"] +)] +#[case( + "fences_compress_tilde_fences", + lines_vec!["~~~~~python", "print('hello')", "~~~~~"] +)] +#[case( + "fences_preserve_interior_conflict", + lines_vec!["````markdown", "```rust", "fn main() {}", "```", "````"] +)] +fn fence_normalization_snapshots(#[case] name: &str, #[case] input: Vec) { + assert_fence_snapshot(name, &input); +} #[test] fn compresses_backtick_fences() { let input = lines_vec!["````rust", "code", "````"]; @@ -27,6 +53,16 @@ fn compresses_tilde_fences() { assert_eq!(out, lines_vec!["```rust", "code", "```"]); } +#[test] +fn preserves_null_language_during_interior_conflict() { + let input = lines_vec!["````null", "```rust", "fn main() {}", "```", "````"]; + let out = compress_fences(&input); + + assert_eq!( + out, + lines_vec!["````", "```rust", "fn main() {}", "```", "````"] + ); +} #[rstest] #[case( lines_vec!["````markdown", "```rust", "fn main() {}", "```", "````"], diff --git a/tests/snapshots/fences_compress_overlong_backticks.snap b/tests/snapshots/fences_compress_overlong_backticks.snap new file mode 100644 index 00000000..3fb62bd0 --- /dev/null +++ b/tests/snapshots/fences_compress_overlong_backticks.snap @@ -0,0 +1,7 @@ +--- +source: tests/fences.rs +expression: "compress_fences(&input).join(\"\\n\")" +--- +```rust +fn main() {} +``` diff --git a/tests/snapshots/fences_compress_tilde_fences.snap b/tests/snapshots/fences_compress_tilde_fences.snap new file mode 100644 index 00000000..ca3478ee --- /dev/null +++ b/tests/snapshots/fences_compress_tilde_fences.snap @@ -0,0 +1,7 @@ +--- +source: tests/fences.rs +expression: "compress_fences(&input).join(\"\\n\")" +--- +```python +print('hello') +``` diff --git a/tests/snapshots/fences_preserve_interior_conflict.snap b/tests/snapshots/fences_preserve_interior_conflict.snap new file mode 100644 index 00000000..d45411a3 --- /dev/null +++ b/tests/snapshots/fences_preserve_interior_conflict.snap @@ -0,0 +1,9 @@ +--- +source: tests/fences.rs +expression: "compress_fences(&input).join(\"\\n\")" +--- +````markdown +```rust +fn main() {} +``` +````