fix: better spacing

Author: harlan-zw

scripts/crawl.ts

@@ -13,7 +13,9 @@ const crawler = new PlaywrightCrawler({
     const html = await page.innerHTML('html')
     log.info('HTML length', { url: request.loadedUrl, length: html.length })
     const now = new Date()
-    const md = syncHtmlToMarkdown(html, withMinimalPreset())
+    const md = syncHtmlToMarkdown(html, withMinimalPreset({
+      origin: new URL(request.loadedUrl).origin,
+    }))
     log.info('Processed html -> md in', { url: request.loadedUrl, time: new Date() - now })
     // mkdir
     if (!existsSync('./output')) {

src/buffer-region.ts

@@ -85,7 +85,7 @@ export function assembleBufferedContent(
   const fragments: string[] = []
 
   // Then process all other regions (excluding frontmatter)
-  for (const [regionId, content] of state.regionContentBuffers) {
+  for (const [regionId, content] of Array.from(state.regionContentBuffers.entries())) {
     // Check if region should be included
     const include = state.regionToggles.get(regionId)
     if (include) {

src/index.ts

@@ -6,9 +6,9 @@ export function syncHtmlToMarkdown(
   options: HTMLToMarkdownOptions = {},
 ): string {
   // Initialize state
-  const state: MdreamRuntimeState = {
+  const state = {
     options,
-  }
+  } as MdreamRuntimeState
   const result = processPartialHTMLToMarkdown(html, state).chunk
   return result.trimEnd()
 }

src/markdown.ts

@@ -1,4 +1,27 @@
 import type { ElementNode, HandlerContext, MdreamRuntimeState, NodeEvent, TextNode } from './types'
+
+/**
+ * Determines if spacing is needed between two characters
+ */
+function needsSpacing(lastChar: string, firstChar: string): boolean {
+  const noSpaceLastChars = new Set(['\n', ' ', '[', '>', '_', '*', '`', '|', '#', '<', '('])
+  const noSpaceFirstChars = new Set([' ', '\n', '\t', '_', '*', '`', '|', '>', '#'])
+  
+  return !noSpaceLastChars.has(lastChar) && !noSpaceFirstChars.has(firstChar)
+}
+
+/**
+ * Determines if spacing should be added before text content
+ */
+function shouldAddSpacingBeforeText(lastChar: string, lastNode: any, textNode: TextNode): boolean {
+  return lastChar && 
+         lastChar !== '\n' && 
+         lastChar !== ' ' && 
+         lastChar !== '[' && 
+         lastChar !== '>' && 
+         !lastNode?.tagHandler?.isInline && 
+         textNode.value[0] !== ' '
+}
 import { collectNodeContent } from './buffer-region'
 import {
   DEFAULT_BLOCK_SPACING,
@@ -41,11 +64,23 @@ export function processHtmlEventToMarkdown(
   state: MdreamRuntimeState,
 ): void {
   const { type: eventType, node } = event
-
+  const lastNode = state.lastNode
+  state.lastNode = event.node
+  const buff = state.regionContentBuffers.get(node.regionId || 0) || []
+  const lastBuffEntry = buff[buff.length - 1]
+  const lastChar = lastBuffEntry?.charAt(lastBuffEntry.length - 1) || ''
+  // we need to see if it exists within buff[lastIndex] or buff[lastIndex - 1]
+  let secondLastChar
+  if (lastBuffEntry?.length > 1) {
+    secondLastChar = lastBuffEntry.charAt(lastBuffEntry.length - 2)
+  }
+  else {
+    secondLastChar = buff[buff.length - 2]?.charAt(buff[buff.length - 2].length - 1)
+  }
   // Handle text nodes
   if (node.type === TEXT_NODE && eventType === NodeEventEnter) {
     const textNode = node as TextNode
-    state.lastNewLines = 0
+
     if (textNode.value) {
       // Process text node with plugins
       if (state.plugins?.length) {
@@ -58,6 +93,16 @@ export function processHtmlEventToMarkdown(
           textNode.value = pluginResult.content
         }
       }
+      // Skip leading spaces after newlines
+      if (textNode.value === ' ' && lastChar === '\n') {
+        return
+      }
+      
+      // Add spacing before text if needed
+      if (shouldAddSpacingBeforeText(lastChar, lastNode, textNode)) {
+        textNode.value = ` ${textNode.value}`
+      }
+      
       collectNodeContent(textNode, textNode.value, state)
     }
     state.lastTextNode = textNode
@@ -89,6 +134,14 @@ export function processHtmlEventToMarkdown(
     output.push(...results)
   }
 
+  let lastNewLines = 0
+  if (lastChar === '\n') {
+    lastNewLines++
+  }
+  if (secondLastChar === '\n') {
+    lastNewLines++
+  }
+
   const eventFn = eventType === NodeEventEnter ? 'enter' : 'exit'
   // Use the cached tag handler directly from the node
   const handler = node.tagHandler
@@ -99,82 +152,58 @@ export function processHtmlEventToMarkdown(
     }
   }
 
-  // Trim trailing whitespace from the last text node
-  if (!state.lastNewLines && lastFragment && state.lastTextNode?.containsWhitespace && !!node.parent && 'value' in state.lastTextNode && typeof state.lastTextNode.value === 'string') {
-    if (!node.parent.depthMap[TAG_PRE] || node.parent.tagId === TAG_PRE) {
-      const originalLength = lastFragment.length
-      const trimmed = lastFragment.trimEnd()
-      const trimmedChars = originalLength - trimmed.length
-
-      // Update the last content in buffer regions with trimmed content
-      if (trimmedChars > 0) {
-        for (const buffer of Array.from(state.regionContentBuffers.values())) {
-          if (buffer.length > 0 && buffer[buffer.length - 1] === lastFragment) {
-            buffer[buffer.length - 1] = trimmed
-            break
-          }
-        }
-      }
-
-      state.lastTextNode = undefined
-    }
-  }
-
   // Handle newlines
   const newLineConfig = calculateNewLineConfig(node as ElementNode)
-  let newLines = newLineConfig[eventType] || 0
+  const newLines = Math.max(0, (newLineConfig[eventType] || 0) - lastNewLines)
 
   if (newLines > 0) {
-    // Initialize lastNewLines if undefined
-    state.lastNewLines ??= 0
-
-    // Adjust count based on existing newlines
-    newLines = Math.max(0, newLines - state.lastNewLines)
+    // if the region has no content, add the current content (without new lines)
+    if (!buff.length) {
+      for (const fragment of output) {
+        collectNodeContent(node, fragment, state)
+      }
+      return
+    }
 
-    // Handle enter events with content
-    if (eventType === NodeEventEnter && output.length) {
-      state.lastNewLines = 0
+    // Add newlines
+    const newlinesStr = '\n'.repeat(newLines)
+    // trim only whitespace
+    if (lastChar === ' ' && buff?.length) {
+      buff[buff.length - 1] = buff[buff.length - 1].substring(0, buff[buff.length - 1].length - 1)
     }
-    if (newLines > 0) {
-      if (!state.regionContentBuffers.get(event.node.regionId || 0)?.length) {
-        for (const fragment of output) {
-          collectNodeContent(node, fragment, state)
-        }
-        return
-      }
-      // Update state for non-enter events
-      if (eventType !== NodeEventEnter || !output.length) {
-        state.lastNewLines = newLines
-      }
 
-      // Add newlines
-      const newlinesStr = '\n'.repeat(newLines)
-      // trim only whitespace
-      if (lastFragment && typeof lastFragment === 'string' && lastFragment.length > 0) {
-        const lastChar = lastFragment.charAt(lastFragment.length - 1)
-        if (lastChar === ' ') {
-          // Update the last content in buffer regions with trimmed content
-          for (const buffer of Array.from(state.regionContentBuffers.values())) {
-            if (buffer.length > 0 && buffer[buffer.length - 1] === lastFragment) {
-              buffer[buffer.length - 1] = lastFragment.substring(0, lastFragment.length - 1)
-              break
-            }
+    if (eventType === NodeEventEnter) {
+      output.unshift(newlinesStr)
+    }
+    else {
+      output.push(newlinesStr)
+    }
+  }
+  else {
+    // trim whitespaced between inline output
+    // Trim trailing whitespace from the last text node
+    if (lastFragment && state.lastTextNode?.containsWhitespace && !!node.parent && 'value' in state.lastTextNode && typeof state.lastTextNode.value === 'string') {
+      if (!node.parent.depthMap[TAG_PRE] || node.parent.tagId === TAG_PRE) {
+        const originalLength = lastFragment.length
+        const trimmed = lastFragment.trimEnd()
+        const trimmedChars = originalLength - trimmed.length
+
+        // Update the last content in buffer regions with trimmed content
+        if (trimmedChars > 0) {
+          if (buff?.length && buff[buff.length - 1] === lastFragment) {
+            buff[buff.length - 1] = trimmed
           }
         }
-      }
 
-      if (eventType === NodeEventEnter) {
-        output.unshift(newlinesStr)
-      }
-      else {
-        output.push(newlinesStr)
+        state.lastTextNode = undefined
       }
     }
   }
-  else {
-    state.lastNewLines = 0
-  }
 
+  // Add spacing between inline elements if needed
+  if (output[0]?.[0] && eventType === NodeEventEnter && lastChar && needsSpacing(lastChar, output[0][0])) {
+    collectNodeContent(node, ' ', state)
+  }
   // Calculate total length of output fragments before adding to the main fragments
   for (const fragment of output) {
     collectNodeContent(node, fragment, state)