Use custom parser for markdown processing.

mdoc-docs/src/main/scala/mdoc/docs/MdocModifier.scala

@@ -23,8 +23,16 @@ class MdocModifier(context: Context) extends StringModifier {
     myStdout.reset()
     myReporter.reset()
     val cleanInput = Input.VirtualFile(code.filename, code.text)
-    val markdown = Markdown.toMarkdown(cleanInput, markdownSettings, myReporter, context.settings)
-    val links = DocumentLinks.fromMarkdown(GitHubIdGenerator, RelativePath("readme.md"), cleanInput)
+    val relpath = RelativePath("readme.md")
+    val markdown = Markdown.toMarkdown(
+      cleanInput,
+      context,
+      relpath,
+      Map.empty[String, String],
+      myReporter,
+      context.settings
+    )
+    val links = DocumentLinks.fromMarkdown(GitHubIdGenerator, relpath, cleanInput)
     LinkHygiene.lint(List(links), myReporter, verbose = false)
     val stdout = fansi.Str(myStdout.toString()).plainText
     if (myReporter.hasErrors || myReporter.hasWarnings) {

mdoc-sbt/src/sbt-test/sbt-mdoc/basic/build.sbt

@@ -18,11 +18,8 @@ println(example.Example.greeting)
 ```scala
 println("Hello Scala.js!")
 ```
-
 <div id="mdoc-js-run0" data-mdoc-js></div>
-
 <script type="text/javascript" src="readme.md.js" defer></script>
-
 <script type="text/javascript" src="mdoc.js" defer></script>
 """.trim,
     "\"\"\"\n" + obtained + "\n\"\"\""

mdoc/src/main/scala/mdoc/internal/cli/MainOps.scala

@@ -24,10 +24,10 @@ import scala.meta.io.AbsolutePath
 import scala.util.control.NonFatal
 
 final class MainOps(
-    settings: Settings,
-    markdown: MutableDataSet,
-    reporter: Reporter
+    context: Context
 ) {
+  def settings: Settings = context.settings
+  def reporter: Reporter = context.reporter
 
   private var livereload: Option[LiveReload] = None
   private def startLivereload(): Unit = {
@@ -59,9 +59,7 @@ final class MainOps(
     val timer = new Timer
     val source = FileIO.slurp(file.in, settings.charset)
     val input = Input.VirtualFile(file.in.toString(), source)
-    markdown.set(Markdown.InputKey, Some(input))
-    markdown.set(Markdown.RelativePathKey, Some(file.relpath))
-    val md = Markdown.toMarkdown(input, markdown, reporter, settings)
+    val md = Markdown.toMarkdown(input, context, file.relpath, settings.site, reporter, settings)
     val fileHasErrors = reporter.errorCount > originalErrors
     if (!fileHasErrors) {
       writePath(file, md)
@@ -226,11 +224,10 @@ object MainOps {
             error.all.foreach(message => reporter.error(message))
             1
           case Configured.Ok(ctx) =>
-            val markdown = Markdown.mdocSettings(ctx)
             if (ctx.settings.verbose) {
               ctx.reporter.setDebugEnabled(true)
             }
-            val runner = new MainOps(ctx.settings, markdown, ctx.reporter)
+            val runner = new MainOps(ctx)
             val exit = runner.run()
             if (exit.isSuccess) {
               0

mdoc/src/main/scala/mdoc/internal/markdown/FenceInput.scala

@@ -0,0 +1,101 @@
+package mdoc.internal.markdown
+
+import com.vladsch.flexmark.ast.FencedCodeBlock
+import scala.meta.inputs.Input
+import scala.meta.inputs.Position
+import mdoc.internal.cli.Context
+import mdoc.internal.markdown.Modifier.Str
+import mdoc.internal.markdown.Modifier.Default
+import mdoc.internal.markdown.Modifier.Post
+import mdoc.internal.markdown.Modifier.Pre
+
+case class PreFenceInput(block: CodeFence, input: Input, mod: Pre)
+case class StringFenceInput(block: CodeFence, input: Input, mod: Str)
+case class ScalaFenceInput(block: CodeFence, input: Input, mod: Modifier)
+
+class FenceInput(ctx: Context, baseInput: Input) {
+  def getModifier(info: Text): Option[Modifier] = {
+    val string = info.value.stripLineEnd
+    if (!string.startsWith("scala mdoc")) None
+    else {
+      if (!string.contains(':')) Some(Modifier.Default())
+      else {
+        val mode = string.stripPrefix("scala mdoc:")
+        Modifier(mode)
+          .orElse {
+            val (name, info) = mode.split(":", 2) match {
+              case Array(a) => (a, "")
+              case Array(a, b) => (a, b)
+            }
+            ctx.settings.stringModifiers
+              .collectFirst[Modifier] {
+                case mod if mod.name == name =>
+                  Str(mod, info)
+              }
+              .orElse {
+                ctx.settings.postModifiers.collectFirst {
+                  case mod if mod.name == name =>
+                    Post(mod, info)
+                }
+              }
+              .orElse {
+                ctx.settings.preModifiers.collectFirst {
+                  case mod if mod.name == name =>
+                    Pre(mod, info)
+                }
+              }
+          }
+          .orElse {
+            invalid(info, s"Invalid mode '$mode'")
+            None
+          }
+      }
+    }
+  }
+
+  private def invalid(info: Text, message: String): Unit = {
+    val offset = "scala mdoc:".length
+    val start = info.pos.start + offset
+    val end = info.pos.end - 1
+    val pos = Position.Range(baseInput, start, end)
+    ctx.reporter.error(pos, message)
+  }
+  private def invalidCombination(info: Text, mod1: String, mod2: String): Boolean = {
+    invalid(info, s"invalid combination of modifiers '$mod1' and '$mod2'")
+    false
+  }
+
+  private def isValid(info: Text, mod: Modifier): Boolean = {
+    if (mod.isFail && mod.isCrash) {
+      invalidCombination(info, "crash", "fail")
+    } else if (mod.isSilent && mod.isInvisible) {
+      invalidCombination(info, "silent", "invisible")
+    } else if (mod.isCompileOnly) {
+      val others = mod.mods - Mod.CompileOnly
+      if (others.isEmpty) {
+        true
+      } else {
+        val all = others.map(_.toString.toLowerCase).mkString(", ")
+        invalid(
+          info,
+          s"""compile-only cannot be used in combination with $all"""
+        )
+        false
+      }
+    } else {
+      true
+    }
+  }
+  def unapply(block: CodeFence): Option[ScalaFenceInput] = {
+    getModifier(block.info) match {
+      case Some(mod) =>
+        if (isValid(block.info, mod)) {
+          val input = Input.Slice(baseInput, block.body.pos.start, block.body.pos.end)
+          Some(ScalaFenceInput(block, input, mod))
+        } else {
+          None
+        }
+      case _ => None
+    }
+  }
+}

mdoc/src/main/scala/mdoc/internal/markdown/Markdown.scala

@@ -99,6 +99,26 @@ object Markdown {
     )
   }
 
+  def toMarkdown(
+      input: Input,
+      context: Context,
+      relativePath: RelativePath,
+      siteVariables: Map[String, String],
+      reporter: Reporter,
+      settings: Settings
+  ): String = {
+    val textWithVariables = VariableRegex.replaceVariables(
+      input,
+      siteVariables,
+      reporter,
+      settings
+    )
+    val file = MarkdownFile.parse(textWithVariables, relativePath, reporter)
+    val processor = new Processor()(context)
+    processor.processDocument(file)
+    file.renderToString
+  }
+
   def toMarkdown(
       input: Input,
       markdownSettings: MutableDataSet,

mdoc/src/main/scala/mdoc/internal/markdown/MarkdownFile.scala

@@ -0,0 +1,128 @@
+package mdoc.internal.markdown
+
+import scala.meta.inputs.Position
+import scala.meta.inputs.Input
+import mdoc.Reporter
+import scala.collection.mutable
+import scala.tools.nsc.doc.DocParser
+import scala.meta.io.RelativePath
+
+final case class MarkdownFile(input: Input, relativePath: RelativePath, parts: List[MarkdownPart]) {
+  private val appends = mutable.ListBuffer.empty[String]
+  def appendText(text: String): Unit = {
+    appends += text
+  }
+  def renderToString: String = {
+    val out = new StringBuilder()
+    parts.foreach(_.renderToString(out))
+    appends.foreach(a => out.append(a).append("\n"))
+    out.toString()
+  }
+}
+object MarkdownFile {
+  sealed abstract class State
+  object State {
+    case class CodeFence(start: Int, backticks: String, info: String) extends State
+    case object Text extends State
+  }
+  class Parser(input: Input, reporter: Reporter) {
+    private val text = input.text
+    private def newPos(start: Int, end: Int): Position = {
+      Position.Range(input, start, end)
+    }
+    private def newText(start: Int, end: Int): Text = {
+      val adaptedEnd = math.max(start, end)
+      val part = Text(text.substring(start, adaptedEnd))
+      part.pos = newPos(start, adaptedEnd)
+      part
+    }
+    private def newCodeFence(
+        state: State.CodeFence,
+        backtickStart: Int,
+        backtickEnd: Int
+    ): CodeFence = {
+      val open = newText(state.start, state.start + state.backticks.length())
+      val info = newText(open.pos.end, open.pos.end + state.info.length())
+      val adaptedBacktickStart = math.max(0, backtickStart - 1)
+      val body = newText(info.pos.end, adaptedBacktickStart)
+      val close = newText(adaptedBacktickStart, backtickEnd)
+      val part = CodeFence(open, info, body, close)
+      part.pos = newPos(state.start, backtickEnd)
+      part
+    }
+    def acceptParts(): List[MarkdownPart] = {
+      var state: State = State.Text
+      val parts = mutable.ListBuffer.empty[MarkdownPart]
+      var curr = 0
+      text.linesWithSeparators.foreach { line =>
+        val end = curr + line.length()
+        state match {
+          case State.Text =>
+            if (line.startsWith("```")) {
+              val backticks = line.takeWhile(_ == '`')
+              val info = line.substring(backticks.length())
+              state = State.CodeFence(curr, backticks, info)
+            } else {
+              parts += newText(curr, end)
+            }
+          case s: State.CodeFence =>
+            if (line.startsWith(s.backticks) &&
+              line.forall(ch => ch == '`' || ch.isWhitespace)) {
+              parts += newCodeFence(s, curr, end)
+              state = State.Text
+            }
+        }
+        curr += line.length()
+      }
+      state match {
+        case s: State.CodeFence =>
+          parts += newCodeFence(s, text.length(), text.length())
+        case _ =>
+      }
+      parts.toList
+    }
+  }
+  def parse(input: Input, relativePath: RelativePath, reporter: Reporter): MarkdownFile = {
+    val parser = new Parser(input, reporter)
+    val parts = parser.acceptParts()
+    MarkdownFile(input, relativePath, parts)
+  }
+}
+
+sealed abstract class MarkdownPart {
+  var pos: Position = Position.None
+  final def renderToString(out: StringBuilder): Unit = this match {
+    case Text(value) =>
+      out.append(value)
+    case fence: CodeFence =>
+      fence.newPart match {
+        case Some(newPart) =>
+          out.append(newPart)
+        case None =>
+          fence.openBackticks.renderToString(out)
+          fence.newInfo match {
+            case None =>
+              fence.info.renderToString(out)
+            case Some(newInfo) =>
+              out.append(newInfo)
+              if (!newInfo.endsWith("\n")) {
+                out.append("\n")
+              }
+          }
+          fence.newBody match {
+            case None =>
+              fence.body.renderToString(out)
+            case Some(newBody) =>
+              out.append(newBody)
+          }
+          fence.closeBackticks.renderToString(out)
+      }
+  }
+}
+final case class Text(value: String) extends MarkdownPart
+final case class CodeFence(openBackticks: Text, info: Text, body: Text, closeBackticks: Text)
+    extends MarkdownPart {
+  var newPart = Option.empty[String]
+  var newInfo = Option.empty[String]
+  var newBody = Option.empty[String]
+}

mdoc/src/main/scala/mdoc/internal/markdown/MdocExtensions.scala

@@ -24,10 +24,7 @@ object MdocExtensions {
     * @return A sequence of extensions to be applied to Flexmark's options.
     */
   def mdoc(context: Context): List[Extension] = {
-    plain ++ List(
-      MdocParserExtension.create(context),
-      MdocFormatterExtension.create(context.settings)
-    )
+    plain
   }
 
   def plain: List[Extension] = {