Merge pull request #9 from DongyuZhao/codex/add-backslash-escapes-and-special-character-handling

Implement escaped character support in Markdown tokenizer

Author: DongyuZhao

Sources/SwiftParser/Languages/MarkdownLanguage.swift

@@ -126,7 +126,17 @@ public struct MarkdownLanguage: CodeLanguage {
             func add(_ t: Token) { tokens.append(t) }
             while index < input.endIndex {
                 let ch = input[index]
-                if ch == "#" {
+                if ch == "\\" {
+                    let start = index
+                    advance()
+                    if index < input.endIndex {
+                        let escaped = input[index]
+                        advance()
+                        add(.text(String(escaped), start..<index))
+                    } else {
+                        add(.text("\\", start..<index))
+                    }
+                } else if ch == "#" {
                     let start = index
                     advance()
                     add(.hash(start..<index))
@@ -215,7 +225,7 @@ public struct MarkdownLanguage: CodeLanguage {
                     let start = index
                     while index < input.endIndex &&
                           input[index] != "\n" &&
-                          !"#-*+_`[].()<>!~|;&=".contains(input[index]) &&
+                          !"#-*+_`[].()<>!~|;&=\\".contains(input[index]) &&
                           !input[index].isNumber {
                         advance()
                     }

Tests/SwiftParserTests/SwiftParserTests.swift

@@ -92,6 +92,15 @@ final class SwiftParserTests: XCTestCase {
         XCTAssertEqual(result.root.children.first?.type as? MarkdownLanguage.Element, .image)
     }
 
+    func testMarkdownEscapedCharacters() {
+        let parser = SwiftParser()
+        let source = "\\*not italic\\*"
+        let result = parser.parse(source, language: MarkdownLanguage())
+        XCTAssertEqual(result.errors.count, 0)
+        XCTAssertEqual(result.root.children.count, 1)
+        XCTAssertEqual(result.root.children.first?.value, "*not italic*")
+    }
+
     func testPrattExpression() {
         let parser = SwiftParser()
         let source = "x = 1 + 2 * 3"