Merge pull request #101 from DongyuZhao/copilot/fix-da3a1cd3-7bcb-496f-ad74-3ceee87548b1

Fix soft line break processing in Markdown paragraph and blockquote parsing

Author: DongyuZhao

Sources/CodeParserCollection/Markdown/MarkdownConstructState.swift

@@ -1,51 +1,25 @@
 import CodeParserCore
 import Foundation
 
-/// Main construction state for Markdown language with line-based processing
+/// Minimal construction state for Markdown language
+/// Only contains state that cannot be derived from the AST (context.current)
 public class MarkdownConstructState: CodeConstructState {
   public typealias Node = MarkdownNodeElement
   public typealias Token = MarkdownTokenElement
 
-  // Current token index in the line
-  public var position: Int = 0
-  // Flag indicates if the block builders should run another round on the same line.
-  public var refreshed: Bool = false
-  // Flag indicates if the current line is being reprocessed after partial consumption
-  public var isPartialLine: Bool = false
-  
-  // Fenced code block state
-  public var openFence: OpenFenceInfo?
-  
-  // HTML block state
-  public var openHTMLBlock: OpenHTMLBlockInfo?
-  
-  /// Stack for nested list processing
-  public var listStack: [ListNode] = []
-  public var currentDefinitionList: DefinitionListNode?
-  
-  /// Enhanced list context tracking for better indentation and nesting management
-  public var listContextStack: [ListContextInfo] = []
-
-  /// Indicates the last consumed line break formed a blank line (two or more consecutive newlines)
-  public var lastWasBlankLine: Bool = false
-
-  /// When a quoted blank line (`>\\n`) is seen inside a blockquote, the next quoted
-  /// content should start a new paragraph inside the same blockquote instead of
-  /// merging into the previous one.
-  public var pendingBlockquoteParagraphSplit: Bool = false
-
-  /// True when the previous quoted line (inside a blockquote) began with a token
-  /// that could start a block (e.g., `#`, `-`, `*`, `+`, number.). We use this to
-  /// prevent merging the next quoted line into the same paragraph, matching CommonMark
-  /// semantics where block-starting constructs introduce a new block.
-  public var prevBlockquoteLineWasBlockStart: Bool = false
-
   /// Reference link definitions storage for resolving reference links
   /// Key is normalized reference identifier (case-insensitive, whitespace collapsed)
+  /// Note: This cannot be derived from AST since reference definitions may appear
+  /// anywhere in the document and need to be available for link resolution
   public var referenceDefinitions: [String: (url: String, title: String)] = [:]
-
-  /// Pending reference link definition being parsed across multiple lines
-  public var pendingReference: PendingReferenceDefinition?
+  
+  /// Current line tokens being processed - builders can modify these
+  /// This allows builders to consume their part and leave remaining tokens for further processing
+  public var tokens: [any CodeToken<MarkdownTokenElement>] = []
+  
+  /// Flag indicating if current line has been fully processed by a builder
+  /// When false, MarkdownBlockBuilder should continue processing the remaining tokens
+  public var currentLineProcessed: Bool = true
 
   public init() {}
 
@@ -72,85 +46,3 @@ public class MarkdownConstructState: CodeConstructState {
       .trimmingCharacters(in: .whitespacesAndNewlines)
   }
 }
-
-/// Information about a pending reference link definition being parsed across multiple lines
-public struct PendingReferenceDefinition {
-  public let identifier: String
-  public let referenceNode: ReferenceNode
-  public var hasDestination: Bool
-  public var hasTitle: Bool
-  public let originalLineTokens: [any CodeToken<MarkdownTokenElement>] // For fallback to paragraph
-  
-  public init(identifier: String, referenceNode: ReferenceNode, originalLineTokens: [any CodeToken<MarkdownTokenElement>]) {
-    self.identifier = identifier
-    self.referenceNode = referenceNode
-    self.hasDestination = false
-    self.hasTitle = false
-    self.originalLineTokens = originalLineTokens
-  }
-}
-
-/// Information about an open fenced code block
-public struct OpenFenceInfo {
-  public let character: String
-  public let length: Int
-  public let indentation: Int
-  public let codeBlock: CodeBlockNode
-  
-  public init(character: String, length: Int, indentation: Int, codeBlock: CodeBlockNode) {
-    self.character = character
-    self.length = length
-    self.indentation = indentation
-    self.codeBlock = codeBlock
-  }
-}
-
-/// Information about an open HTML block
-public struct OpenHTMLBlockInfo {
-  public let type: Int // HTML block type (1-7)
-  public let endCondition: String? // What string ends this block
-  public let htmlBlock: HTMLBlockNode
-  
-  public init(type: Int, endCondition: String?, htmlBlock: HTMLBlockNode) {
-    self.type = type
-    self.endCondition = endCondition
-    self.htmlBlock = htmlBlock
-  }
-}
-
-/// Information about detected HTML block type
-public struct HTMLBlockTypeInfo {
-  public let type: Int
-  public let name: String
-  public let closedOnSameLine: Bool
-  public let endCondition: String?
-  
-  public init(type: Int, name: String, closedOnSameLine: Bool, endCondition: String? = nil) {
-    self.type = type
-    self.name = name
-    self.closedOnSameLine = closedOnSameLine
-    self.endCondition = endCondition
-  }
-}
-
-/// Enhanced list context information for better nesting and indentation management
-public struct ListContextInfo {
-  /// The list node itself
-  public let list: ListNode
-  /// The parent list item that contains this list (nil for top-level lists)
-  public let parentListItem: ListItemNode?
-  /// The calculated indentation level for content in this list context
-  public let contentIndent: Int
-  /// The nesting level (1 for top-level, 2 for first nested, etc.)
-  public let level: Int
-  /// The marker type for compatibility checking
-  public let markerType: String
-  
-  public init(list: ListNode, parentListItem: ListItemNode?, contentIndent: Int, level: Int, markerType: String) {
-    self.list = list
-    self.parentListItem = parentListItem
-    self.contentIndent = contentIndent
-    self.level = level
-    self.markerType = markerType
-  }
-}

Sources/CodeParserCollection/Markdown/MarkdownNodes.swift

@@ -110,13 +110,16 @@ public class DocumentNode: MarkdownNodeBase {
 }
 
 // MARK: - Block Elements
-public class ParagraphNode: MarkdownNodeBase {
+public class ParagraphNode: MarkdownNodeBase, MarkdownBlockNode {
+  public var blockType: String { "paragraph" }
+  
   public init(range: Range<String.Index>) {
     super.init(element: .paragraph)
   }
 }
 
-public class HeaderNode: MarkdownNodeBase {
+public class HeaderNode: MarkdownNodeBase, MarkdownBlockNode {
+  public var blockType: String { "heading" }
   public var level: Int
 
   public init(level: Int) {
@@ -130,7 +133,8 @@ public class HeaderNode: MarkdownNodeBase {
   }
 }
 
-public class ThematicBreakNode: MarkdownNodeBase {
+public class ThematicBreakNode: MarkdownNodeBase, MarkdownBlockNode {
+  public var blockType: String { "thematic_break" }
   public var marker: String
 
   public init(marker: String = "---") {
@@ -144,8 +148,14 @@ public class ThematicBreakNode: MarkdownNodeBase {
   }
 }
 
-public class BlockquoteNode: MarkdownNodeBase {
+public class BlockquoteNode: MarkdownNodeBase, MarkdownBlockNode {
+  public var blockType: String { "blockquote" }
   public var level: Int
+  
+  // Package-level indentation properties for nested block parsing
+  package var indent: Int = 0  // Number of spaces before the '>' marker
+  package var markerColumn: Int = 0  // Column position of the '>' marker
+  package var contentColumn: Int = 0  // Column position where content starts after '> '
 
   public init(level: Int = 1) {
     self.level = level
@@ -198,11 +208,17 @@ public class UnorderedListNode: ListNode {
   }
 }
 
-public class ListItemNode: MarkdownNodeBase {
+public class ListItemNode: MarkdownNodeBase, MarkdownBlockNode {
+  public var blockType: String { "list_item" }
   public var marker: String
   // indentation before marker and content indent column for continuation
   public var markerIndent: Int = 0
   public var contentIndent: Int = 0
+  
+  // Package-level properties for enhanced nested block parsing
+  package var markerColumn: Int = 0  // Exact column position of the marker
+  package var contentColumn: Int = 0  // Exact column position where content starts
+  package var markerLength: Int = 0  // Length of the marker (e.g., "1." = 2, "-" = 1)
 
   public init(marker: String) {
     self.marker = marker
@@ -215,9 +231,13 @@ public class ListItemNode: MarkdownNodeBase {
   }
 }
 
-public class CodeBlockNode: MarkdownNodeBase {
+public class CodeBlockNode: MarkdownNodeBase, MarkdownBlockNode {
+  public var blockType: String { "code_block" }
   public var language: String?
   public var source: String
+  
+  // Package-level indentation properties for nested block parsing
+  package var indent: Int = 0  // Number of spaces before the code block
 
   public init(source: String, language: String? = nil) {
     self.language = language
@@ -635,3 +655,12 @@ public class ContentNode: MarkdownNodeBase {
     super.init(element: .content)
   }
 }
+
+// MARK: - Type Aliases for Block Builders
+public typealias MarkdownHeading = HeaderNode
+public typealias MarkdownThematicBreak = ThematicBreakNode
+public typealias MarkdownText = TextNode
+public typealias MarkdownParagraph = ParagraphNode
+public typealias MarkdownBlockquote = BlockquoteNode
+public typealias MarkdownLineBreak = LineBreakNode
+public typealias MarkdownListItem = ListItemNode