Simplify EOF handling with new builder

Sources/SwiftParser/Core/CodeConstructor.swift

@@ -16,7 +16,10 @@ public class CodeConstructor<Node, Token> where Node: CodeNodeElement, Token: Co
     /// - Parameters:
     ///   - builders: The node builders responsible for producing AST nodes.
     ///   - state: Factory returning the initial parsing state object.
-    public init(builders: [any CodeNodeBuilder<Node, Token>], state: @escaping () -> (any CodeConstructState<Node, Token>)?) {
+    public init(
+        builders: [any CodeNodeBuilder<Node, Token>],
+        state: @escaping () -> (any CodeConstructState<Node, Token>)?
+    ) {
         self.builders = builders
         self.state = state
     }
@@ -30,11 +33,6 @@ public class CodeConstructor<Node, Token> where Node: CodeNodeElement, Token: Co
         var context = CodeConstructContext(current: root, tokens: tokens, state: state())
 
         while context.consuming < context.tokens.count {
-            // Stop at EOF without recording an error
-            if let token = context.tokens[context.consuming] as? MarkdownToken,
-               token.element == .eof {
-                break
-            }
 
             var matched = false
             for node in builders {

Sources/SwiftParser/Core/CodeLanguage.swift

@@ -18,4 +18,12 @@ public protocol CodeLanguage<Node, Token> where Node: CodeNodeElement, Token: Co
 
     /// The function that creates the initial context for tokenization.
     func state() -> (any CodeTokenState<Token>)?
+
+    /// Provide an EOF token if the language requires one.
+    /// - Parameter range: The range where the EOF token should be inserted.
+    func eofToken(at range: Range<String.Index>) -> (any CodeToken<Token>)?
+}
+
+extension CodeLanguage {
+    public func eofToken(at range: Range<String.Index>) -> (any CodeToken<Token>)? { nil }
 }

Sources/SwiftParser/Core/CodeParser.swift

@@ -30,8 +30,15 @@ public class CodeParser<Node: CodeNodeElement, Token: CodeTokenElement> where No
 
     public init(language: any CodeLanguage<Node, Token>) {
         self.language = language
-        self.tokenizer = CodeTokenizer(builders: language.tokens, state: language.state)
-        self.constructor = CodeConstructor(builders: language.nodes, state: language.state)
+        self.tokenizer = CodeTokenizer(
+            builders: language.tokens,
+            state: language.state,
+            eofTokenFactory: { language.eofToken(at: $0) }
+        )
+        self.constructor = CodeConstructor(
+            builders: language.nodes,
+            state: language.state
+        )
     }
 
     /// Parse a source string using the supplied language.

Sources/SwiftParser/Core/CodeTokenizer.swift

@@ -8,10 +8,16 @@
 public class CodeTokenizer<Token> where Token: CodeTokenElement {
     private let builders: [any CodeTokenBuilder<Token>]
     private var state: () -> (any CodeTokenState<Token>)?
+    private let eofTokenFactory: ((Range<String.Index>) -> (any CodeToken<Token>)?)?
 
-    public init(builders: [any CodeTokenBuilder<Token>], state: @escaping () -> (any CodeTokenState<Token>)?) {
+    public init(
+        builders: [any CodeTokenBuilder<Token>],
+        state: @escaping () -> (any CodeTokenState<Token>)?,
+        eofTokenFactory: ((Range<String.Index>) -> (any CodeToken<Token>)?)? = nil
+    ) {
         self.builders = builders
         self.state = state
+        self.eofTokenFactory = eofTokenFactory
     }
 
     public func tokenize(_ input: String) -> ([any CodeToken<Token>], [CodeError]) {
@@ -42,10 +48,9 @@ public class CodeTokenizer<Token> where Token: CodeTokenElement {
             }
         }
 
-        // Automatically append EOF token for Markdown
-        if Token.self == MarkdownTokenElement.self,
-           let eof = MarkdownToken.eof(at: input.endIndex..<input.endIndex) as? any CodeToken<Token> {
-            context.tokens.append(eof)
+        // Append EOF token if provided by the language
+        if let token = eofTokenFactory?(input.endIndex..<input.endIndex) {
+            context.tokens.append(token)
         }
 
         return (context.tokens, context.errors)

Sources/SwiftParser/Markdown/MarkdownLanguage.swift

@@ -37,7 +37,8 @@ public class MarkdownLanguage: CodeLanguage {
             MarkdownListBuilder(),
             MarkdownBlockquoteBuilder(),
             MarkdownParagraphBuilder(),
-            MarkdownNewlineBuilder()
+            MarkdownNewlineBuilder(),
+            MarkdownEOFBuilder()
         ]
     ) {
         self.nodes = consumers
@@ -67,6 +68,10 @@ public class MarkdownLanguage: CodeLanguage {
     public func state() -> (any CodeTokenState<MarkdownTokenElement>)? {
         nil
     }
+
+    public func eofToken(at range: Range<String.Index>) -> (any CodeToken<MarkdownTokenElement>)? {
+        return MarkdownToken.eof(at: range)
+    }
 }
 
 // MARK: - Language Configuration

Sources/SwiftParser/Markdown/Nodes/MarkdownEOFBuilder.swift

@@ -0,0 +1,14 @@
+import Foundation
+
+/// Consumes trailing EOF tokens without modifying the AST.
+public class MarkdownEOFBuilder: CodeNodeBuilder {
+    public init() {}
+
+    public func build(from context: inout CodeConstructContext<MarkdownNodeElement, MarkdownTokenElement>) -> Bool {
+        guard context.consuming < context.tokens.count,
+              let token = context.tokens[context.consuming] as? MarkdownToken,
+              token.element == .eof else { return false }
+        context.consuming += 1
+        return true
+    }
+}

Tests/SwiftParserTests/Markdown/Tokenizer/MarkdownCodeTokenizerBasicTests.swift

@@ -4,7 +4,11 @@ import XCTest
 final class MarkdownCodeTokenizerBasicTests: XCTestCase {
     func testHeadingTokenization() {
         let language = MarkdownLanguage()
-        let tokenizer = CodeTokenizer(builders: language.tokens, state: language.state)
+        let tokenizer = CodeTokenizer(
+            builders: language.tokens,
+            state: language.state,
+            eofTokenFactory: { language.eofToken(at: $0) }
+        )
         let (tokens, _) = tokenizer.tokenize("# Title")
         XCTAssertEqual(tokens.count, 4)
         XCTAssertEqual(tokens[0].element, .hash)
@@ -15,7 +19,11 @@ final class MarkdownCodeTokenizerBasicTests: XCTestCase {
 
     func testAutolinkTokenization() {
         let language = MarkdownLanguage()
-        let tokenizer = CodeTokenizer(builders: language.tokens, state: language.state)
+        let tokenizer = CodeTokenizer(
+            builders: language.tokens,
+            state: language.state,
+            eofTokenFactory: { language.eofToken(at: $0) }
+        )
         let (tokens, _) = tokenizer.tokenize("<https://example.com>")
         XCTAssertEqual(tokens.count, 2)
         XCTAssertEqual(tokens[0].element, .autolink)
@@ -25,7 +33,11 @@ final class MarkdownCodeTokenizerBasicTests: XCTestCase {
 
     func testBareURLTokenization() {
         let language = MarkdownLanguage()
-        let tokenizer = CodeTokenizer(builders: language.tokens, state: language.state)
+        let tokenizer = CodeTokenizer(
+            builders: language.tokens,
+            state: language.state,
+            eofTokenFactory: { language.eofToken(at: $0) }
+        )
         let (tokens, _) = tokenizer.tokenize("https://example.com")
         XCTAssertEqual(tokens.count, 2)
         XCTAssertEqual(tokens[0].element, .url)
@@ -34,7 +46,11 @@ final class MarkdownCodeTokenizerBasicTests: XCTestCase {
 
     func testBareEmailTokenization() {
         let language = MarkdownLanguage()
-        let tokenizer = CodeTokenizer(builders: language.tokens, state: language.state)
+        let tokenizer = CodeTokenizer(
+            builders: language.tokens,
+            state: language.state,
+            eofTokenFactory: { language.eofToken(at: $0) }
+        )
         let (tokens, _) = tokenizer.tokenize("user@example.com")
         XCTAssertEqual(tokens.count, 2)
         XCTAssertEqual(tokens[0].element, .email)

Tests/SwiftParserTests/Markdown/Tokenizer/MarkdownCodeTokenizerCodeTests.swift

@@ -4,7 +4,11 @@ import XCTest
 final class MarkdownCodeTokenizerCodeTests: XCTestCase {
     private func tokenize(_ input: String) -> [any CodeToken<MarkdownTokenElement>] {
         let language = MarkdownLanguage()
-        let tokenizer = CodeTokenizer(builders: language.tokens, state: language.state)
+        let tokenizer = CodeTokenizer(
+            builders: language.tokens,
+            state: language.state,
+            eofTokenFactory: { language.eofToken(at: $0) }
+        )
         let (tokens, _) = tokenizer.tokenize(input)
         return tokens
     }

Tests/SwiftParserTests/Markdown/Tokenizer/MarkdownCodeTokenizerCustomContainerTests.swift

@@ -4,7 +4,11 @@ import XCTest
 final class MarkdownCodeTokenizerCustomContainerTests: XCTestCase {
     private func tokenize(_ input: String) -> [any CodeToken<MarkdownTokenElement>] {
         let language = MarkdownLanguage()
-        let tokenizer = CodeTokenizer(builders: language.tokens, state: language.state)
+        let tokenizer = CodeTokenizer(
+            builders: language.tokens,
+            state: language.state,
+            eofTokenFactory: { language.eofToken(at: $0) }
+        )
         let (tokens, _) = tokenizer.tokenize(input)
         return tokens
     }

Tests/SwiftParserTests/Markdown/Tokenizer/MarkdownCodeTokenizerFormulaTests.swift

@@ -4,7 +4,11 @@ import XCTest
 final class MarkdownCodeTokenizerFormulaTests: XCTestCase {
     private func tokenize(_ input: String) -> [any CodeToken<MarkdownTokenElement>] {
         let language = MarkdownLanguage()
-        let tokenizer = CodeTokenizer(builders: language.tokens, state: language.state)
+        let tokenizer = CodeTokenizer(
+            builders: language.tokens,
+            state: language.state,
+            eofTokenFactory: { language.eofToken(at: $0) }
+        )
         let (tokens, _) = tokenizer.tokenize(input)
         return tokens
     }

Tests/SwiftParserTests/Markdown/Tokenizer/MarkdownCodeTokenizerHTMLTests.swift

@@ -4,7 +4,11 @@ import XCTest
 final class MarkdownCodeTokenizerHTMLTests: XCTestCase {
     private func tokenize(_ input: String) -> [any CodeToken<MarkdownTokenElement>] {
         let language = MarkdownLanguage()
-        let tokenizer = CodeTokenizer(builders: language.tokens, state: language.state)
+        let tokenizer = CodeTokenizer(
+            builders: language.tokens,
+            state: language.state,
+            eofTokenFactory: { language.eofToken(at: $0) }
+        )
         let (tokens, _) = tokenizer.tokenize(input)
         return tokens
     }

Tests/SwiftParserTests/Markdown/Tokenizer/MarkdownTokenizerBasicTests.swift

@@ -8,7 +8,11 @@ final class MarkdownTokenizerBasicTests: XCTestCase {
     override func setUp() {
         super.setUp()
         let language = MarkdownLanguage()
-        tokenizer = CodeTokenizer(builders: language.tokens, state: language.state)
+        tokenizer = CodeTokenizer(
+            builders: language.tokens,
+            state: language.state,
+            eofTokenFactory: { language.eofToken(at: $0) }
+        )
     }
 
     override func tearDown() {

Tests/SwiftParserTests/Markdown/Tokenizer/MarkdownTokenizerComplexTests.swift

@@ -8,7 +8,11 @@ final class MarkdownTokenizerComplexTests: XCTestCase {
     override func setUp() {
         super.setUp()
         let language = MarkdownLanguage()
-        tokenizer = CodeTokenizer(builders: language.tokens, state: language.state)
+        tokenizer = CodeTokenizer(
+            builders: language.tokens,
+            state: language.state,
+            eofTokenFactory: { language.eofToken(at: $0) }
+        )
     }
 
     override func tearDown() {

Tests/SwiftParserTests/Markdown/Tokenizer/MarkdownTokenizerFormulaTests.swift

@@ -8,7 +8,11 @@ final class MarkdownTokenizerFormulaTests: XCTestCase {
     override func setUp() {
         super.setUp()
         let language = MarkdownLanguage()
-        tokenizer = CodeTokenizer(builders: language.tokens, state: language.state)
+        tokenizer = CodeTokenizer(
+            builders: language.tokens,
+            state: language.state,
+            eofTokenFactory: { language.eofToken(at: $0) }
+        )
     }
 
     override func tearDown() {

Tests/SwiftParserTests/Markdown/Tokenizer/MarkdownTokenizerHTMLTests.swift

@@ -8,7 +8,11 @@ final class MarkdownTokenizerHTMLTests: XCTestCase {
     override func setUp() {
         super.setUp()
         let language = MarkdownLanguage()
-        tokenizer = CodeTokenizer(builders: language.tokens, state: language.state)
+        tokenizer = CodeTokenizer(
+            builders: language.tokens,
+            state: language.state,
+            eofTokenFactory: { language.eofToken(at: $0) }
+        )
     }
 
     override func tearDown() {