Skip to content

Commit

Permalink
Markdown support fixes
Browse files Browse the repository at this point in the history
 - Hide markdown support behind a feature flag (`--enable-markdown`)
 - still process explicitly passed Markdown files
 - remove some PoC simplifications and workarounds
 - misc refactors
  • Loading branch information
Gedochao committed Aug 22, 2022
1 parent 12fc411 commit a23fb83
Show file tree
Hide file tree
Showing 18 changed files with 728 additions and 414 deletions.
8 changes: 5 additions & 3 deletions modules/build/src/main/scala/scala/build/CrossSources.scala
Expand Up @@ -149,7 +149,8 @@ object CrossSources {
.flatMap(_.options)
.flatMap(_.internal.extraSourceFiles)
.distinct
val inputsElemFromDirectives = value(resolveInputsFromSources(sourcesFromDirectives))
val inputsElemFromDirectives =
value(resolveInputsFromSources(sourcesFromDirectives, inputs.enableMarkdown))
val preprocessedSourcesFromDirectives = value(preprocessSources(inputsElemFromDirectives))
val allInputs = inputs.add(inputsElemFromDirectives)

Expand Down Expand Up @@ -221,12 +222,13 @@ object CrossSources {
(CrossSources(paths, inMemory, defaultMainClassOpt, resourceDirs, buildOptions), allInputs)
}

private def resolveInputsFromSources(sources: Seq[Positioned[os.Path]]) =
private def resolveInputsFromSources(sources: Seq[Positioned[os.Path]], enableMarkdown: Boolean) =
sources.map { source =>
val sourcePath = source.value
lazy val dir = sourcePath / os.up
lazy val subPath = sourcePath.subRelativeTo(dir)
if (os.isDir(sourcePath)) Right(Inputs.singleFilesFromDirectory(Inputs.Directory(sourcePath)))
if (os.isDir(sourcePath))
Right(Inputs.singleFilesFromDirectory(Inputs.Directory(sourcePath), enableMarkdown))
else if (sourcePath.ext == "scala") Right(Seq(Inputs.ScalaFile(dir, subPath)))
else if (sourcePath.ext == "sc") Right(Seq(Inputs.Script(dir, subPath)))
else if (sourcePath.ext == "java") Right(Seq(Inputs.JavaFile(dir, subPath)))
Expand Down
46 changes: 31 additions & 15 deletions modules/build/src/main/scala/scala/build/Inputs.scala
Expand Up @@ -21,7 +21,8 @@ final case class Inputs(
workspace: os.Path,
baseProjectName: String,
mayAppendHash: Boolean,
workspaceOrigin: Option[WorkspaceOrigin]
workspaceOrigin: Option[WorkspaceOrigin],
enableMarkdown: Boolean
) {

def isEmpty: Boolean =
Expand All @@ -30,7 +31,7 @@ final case class Inputs(
def singleFiles(): Seq[Inputs.SingleFile] =
elements.flatMap {
case f: Inputs.SingleFile => Seq(f)
case d: Inputs.Directory => Inputs.singleFilesFromDirectory(d)
case d: Inputs.Directory => Inputs.singleFilesFromDirectory(d, enableMarkdown)
case _: Inputs.ResourceDirectory => Nil
case _: Inputs.Virtual => Nil
}
Expand All @@ -51,7 +52,7 @@ final case class Inputs(
def flattened(): Seq[Inputs.SingleElement] =
elements.flatMap {
case f: Inputs.SingleFile => Seq(f)
case d: Inputs.Directory => Inputs.singleFilesFromDirectory(d)
case d: Inputs.Directory => Inputs.singleFilesFromDirectory(d, enableMarkdown)
case _: Inputs.ResourceDirectory => Nil
case v: Inputs.Virtual => Seq(v)
}
Expand Down Expand Up @@ -109,7 +110,7 @@ final case class Inputs(
case elem: Inputs.OnDisk =>
val content = elem match {
case dirInput: Inputs.Directory =>
Seq("dir:") ++ Inputs.singleFilesFromDirectory(dirInput)
Seq("dir:") ++ Inputs.singleFilesFromDirectory(dirInput, enableMarkdown)
.map(file => s"${file.path}:" + os.read(file.path))
case resDirInput: Inputs.ResourceDirectory =>
// Resource changes for SN require relinking, so they should also be hashed
Expand Down Expand Up @@ -221,7 +222,10 @@ object Inputs {
final case class VirtualData(content: Array[Byte], source: String)
extends Virtual

def singleFilesFromDirectory(d: Inputs.Directory): Seq[Inputs.SingleFile] = {
def singleFilesFromDirectory(
d: Inputs.Directory,
enableMarkdown: Boolean
): Seq[Inputs.SingleFile] = {
import Ordering.Implicits.seqOrdering
os.walk.stream(d.path, skip = _.last.startsWith("."))
.filter(os.isFile(_))
Expand All @@ -232,7 +236,7 @@ object Inputs {
Inputs.ScalaFile(d.path, p.subRelativeTo(d.path))
case p if p.last.endsWith(".sc") =>
Inputs.Script(d.path, p.subRelativeTo(d.path))
case p if p.last.endsWith(".md") =>
case p if p.last.endsWith(".md") && enableMarkdown =>
Inputs.MarkdownFile(d.path, p.subRelativeTo(d.path))
}
.toVector
Expand Down Expand Up @@ -273,7 +277,8 @@ object Inputs {
validElems: Seq[Element],
baseProjectName: String,
directories: Directories,
forcedWorkspace: Option[os.Path]
forcedWorkspace: Option[os.Path],
enableMarkdown: Boolean
): Inputs = {

assert(validElems.nonEmpty)
Expand Down Expand Up @@ -317,7 +322,8 @@ object Inputs {
workspace,
baseProjectName,
mayAppendHash = needsHash,
workspaceOrigin = Some(workspaceOrigin0)
workspaceOrigin = Some(workspaceOrigin0),
enableMarkdown = enableMarkdown
)
}

Expand Down Expand Up @@ -440,7 +446,8 @@ object Inputs {
scalaSnippetList: List[String],
javaSnippetList: List[String],
acceptFds: Boolean,
forcedWorkspace: Option[os.Path]
forcedWorkspace: Option[os.Path],
enableMarkdown: Boolean
): Either[BuildException, Inputs] = {
val validatedArgs: Seq[Either[String, Seq[Element]]] =
validateArgs(args, cwd, download, stdinOpt, acceptFds)
Expand All @@ -456,7 +463,13 @@ object Inputs {
}.flatten
assert(validElems.nonEmpty)

Right(forValidatedElems(validElems, baseProjectName, directories, forcedWorkspace))
Right(forValidatedElems(
validElems,
baseProjectName,
directories,
forcedWorkspace,
enableMarkdown
))
}
else
Left(new InputsException(invalid.mkString(System.lineSeparator())))
Expand All @@ -474,13 +487,14 @@ object Inputs {
scalaSnippetList: List[String] = List.empty,
javaSnippetList: List[String] = List.empty,
acceptFds: Boolean = false,
forcedWorkspace: Option[os.Path] = None
forcedWorkspace: Option[os.Path] = None,
enableMarkdown: Boolean = false
): Either[BuildException, Inputs] =
if (
args.isEmpty && scriptSnippetList.isEmpty && scalaSnippetList.isEmpty && javaSnippetList.isEmpty
)
defaultInputs().toRight(new InputsException(
"No inputs provided (expected files with .scala or .sc extensions, and / or directories)."
"No inputs provided (expected files with .scala, .sc, .java or .md extensions, and / or directories)."
))
else
forNonEmptyArgs(
Expand All @@ -494,19 +508,21 @@ object Inputs {
scalaSnippetList,
javaSnippetList,
acceptFds,
forcedWorkspace
forcedWorkspace,
enableMarkdown
)

def default(): Option[Inputs] =
None

def empty(workspace: os.Path): Inputs =
def empty(workspace: os.Path, enableMarkdown: Boolean): Inputs =
Inputs(
elements = Nil,
defaultMainClassElement = None,
workspace = workspace,
baseProjectName = "project",
mayAppendHash = true,
workspaceOrigin = None
workspaceOrigin = None,
enableMarkdown = enableMarkdown
)
}
@@ -0,0 +1,122 @@
package scala.build.internal.markdown

import scala.collection.mutable

/** A util for handling of code blocks in Markdown. */
object MarkdownCodeBlockUtil {
private val allowIndentedFence: Boolean = false

/** Representation for a code block contained in Markdown
*
* @param info
* a list of tags tied to a given code block
* @param body
* the code block content
* @param startLine
* starting line on which the code block was defined
* @param endLine
* end line on which the code block was closed
*/
case class MarkdownFence(
info: Seq[String],
body: String,
startLine: Int, // start of fenced body EXCLUDING backticks
endLine: Int // same as above
) {
override def toString: String =
s"Fence[$info, lines $startLine-$endLine]{${body.replace("\n", "\\n")}}"

/** @return
* `true` if this snippet should be ignored, `false` otherwise
*/
def shouldIgnore: Boolean = info.head != "scala" || info.contains("ignore")

/** @return
* `true` if this snippet should have its scope reset, `false` otherwise
*/
def resetScope: Boolean = info.contains("reset")

/** @return
* `true` if this snippet is a test snippet, `false` otherwise
*/
def isTest: Boolean = info.contains("test")

/** @return
* `true` if this snippet is a raw snippet, `false` otherwise
*/
def isRaw: Boolean = info.contains("raw")
}

private case class StartedFence(
info: String,
tickStartLine: Int, // fence start INCLUDING backticks
backticks: String,
indent: Int
)

/** Closes started code-fence
*
* @param started
* [[StartedFence]] representing this code-fence's start
* @param tickEndLine
* number of the line where closing backticks are
* @param lines
* input file sliced into lines
* @return
* [[MarkdownFence]] representing whole closed code-fence
*/
private def closeFence(
started: StartedFence,
tickEndLine: Int,
lines: Array[String]
): MarkdownFence = {
val start: Int = started.tickStartLine + 1
val bodyLines: Array[String] = lines.slice(start, tickEndLine)
MarkdownFence(
started.info.split("\\s+").toList, // strip info by whitespaces
bodyLines.tail.foldLeft(bodyLines.head)((body, line) => body.:++("\n" + line)),
start, // snippet has to begin in the new line
tickEndLine - 1 // ending backticks have to be placed below the snippet
)
}

/** Finds all code snippets in given input
*
* @param md
* Markdown file in a `String` format
* @return
* list of all found snippets
*/
def findFences(md: String): Seq[MarkdownFence] = {
var startedFenceOpt: Option[StartedFence] = None
val fences = mutable.ListBuffer.empty[MarkdownFence]
val lines: Array[String] = md.split("\n\r?")
for (i <- lines.indices) {
val line = lines(i)
startedFenceOpt match {
case Some(s) =>
val start: Int = line.indexOf(s.backticks)
if (start == s.indent && line.forall(c => c == '`' || c.isWhitespace)) {
fences += closeFence(s, i, lines)
startedFenceOpt = None
}
case None =>
val start: Int = line.indexOf("```")
if (start == 0 || (start > 0 && allowIndentedFence)) { // doesn't allow snippet indent
val fence = line.substring(start)
val backticks: String = fence.takeWhile(_ == '`')
val info: String = fence.substring(backticks.length)
startedFenceOpt = Some(StartedFence(info, i, backticks, start))
}
}
}
startedFenceOpt match { // snippet can be ended with EOF
case Some(s) =>
fences += closeFence(s, lines.length, lines)
startedFenceOpt = None
case None =>
}

fences.toList.filter(fence => !fence.shouldIgnore)
}
}

0 comments on commit a23fb83

Please sign in to comment.