Rewrite Docs Generator from Python to Scala (#1729)

.github/workflows/scala.yml

@@ -145,6 +145,9 @@ jobs:
         # uploaded from one of the runners.
         if: runner.os == 'Linux'
         run: sbt --no-colors syntaxJS/fullOptJS
+      - name: Build the docs from standard library sources.
+        if: runner.os == 'Linux'
+        run: sbt --no-colors docs-generator/run
 
       # Prepare distributions
       # The version used in filenames is based on the version of the launcher.
@@ -292,6 +295,17 @@ jobs:
         with:
           name: Parser JS Bundle
           path: ./target/scala-parser.js
+      #      - name: Publish the standard library docs
+      #        if: runner.os == 'Linux'
+      #        uses: andstor/copycat-action@v3
+      #        with:
+      #          personal_token: ${{ secrets.CI_PAT }}
+      #          src_path: ./distribution/std-lib/docs-js/
+      #          dst_path: /docs/reference/.
+      #          dst_branch: stdlib-update
+      #          dst_owner: enso-org
+      #          dst_repo_name: website
+      #          clean: true
       - name: Publish the Manifest
         if: runner.os == 'Linux'
         uses: actions/upload-artifact@v2

.gitignore

@@ -70,6 +70,8 @@ package-lock.json
 
 javadoc/
 scaladoc/
+distribution/std-lib/docs
+distribution/std-lib/docs-js
 
 #######################
 ## Benchmark Reports ##

RELEASES.md

@@ -33,6 +33,10 @@
 - Overhauled the types we use for errors throughout the standard library
   ([#1734](https://github.com/enso-org/enso/pull/1734)). They are now much more
   informative, and should provide more clarity when things go wrong.
+- Re-wrote the documentation generator for the Enso website from Python into
+  Scala ([#1729](https://github.com/enso-org/enso/pull/1729)). This has greatly
+  improved the performance, enabling us to generate the documentation structure
+  for the entire standard library 8-10 times faster than before.
 
 ## Miscellaneous
 

build.sbt

@@ -180,6 +180,7 @@ lazy val enso = (project in file("."))
     `json-rpc-server`,
     `language-server`,
     `parser-service`,
+    `docs-generator`,
     `polyglot-api`,
     `project-manager`,
     `syntax-definition`.jvm,
@@ -539,6 +540,16 @@ lazy val `parser-service` = (project in file("lib/scala/parser-service"))
     mainClass := Some("org.enso.ParserServiceMain")
   )
 
+lazy val `docs-generator` = (project in file("lib/scala/docs-generator"))
+  .dependsOn(syntax.jvm)
+  .dependsOn(cli)
+  .settings(
+    libraryDependencies ++= Seq(
+      "commons-cli" % "commons-cli" % commonsCliVersion
+    ),
+    mainClass := Some("org.enso.docs.generator.Main")
+  )
+
 lazy val `text-buffer` = project
   .in(file("lib/scala/text-buffer"))
   .configs(Test)

lib/scala/docs-generator/README.md

@@ -0,0 +1,4 @@
+# Docs Generator
+
+A service generating docs from contents of library, and saving them as `.html`
+files in a `docs` subdirectory.

lib/scala/docs-generator/src/main/scala/org/enso/docs/generator/Constants.scala

@@ -0,0 +1,17 @@
+package org.enso.docs.generator
+
+object Constants {
+  val TEMPLATE_FILES_PATH =
+    "./lib/scala/docs-generator/src/main/scala/org/enso/docs/generator/"
+  val SOURCE_PATH        = "./distribution/std-lib/Standard/src"
+  val JS_TEMPLATE_NAME   = "template.js"
+  val CSS_TREE_FILE_NAME = "treeStyle.css"
+  val OUTPUT_DIRECTORY   = "docs-js"
+
+  val HELP_OPTION          = "help"
+  val INPUT_PATH_OPTION    = "input-path"
+  val OUTPUT_DIR_OPTION    = "output-dir"
+  val DOCS_LIB_PATH_OPTION = "docs-lib-path"
+  val JS_TEMPLATE_OPTION   = "js-template"
+  val CSS_TEMPLATE_OPTION  = "css-template"
+}

lib/scala/docs-generator/src/main/scala/org/enso/docs/generator/DocsGenerator.scala

@@ -0,0 +1,63 @@
+package org.enso.docs.generator
+
+import java.io.File
+import org.enso.syntax.text.{DocParser, Parser}
+import org.enso.syntax.text.docparser._
+
+/** The Docs Generator class. Defines useful wrappers for Doc Parser.
+  */
+object DocsGenerator {
+
+  /** Generates HTML of docs from Enso program.
+    */
+  def run(program: String): String = {
+    val parser   = new Parser()
+    val module   = parser.run(program)
+    val dropMeta = parser.dropMacroMeta(module)
+    val doc      = DocParserRunner.createDocs(dropMeta)
+    val code     = DocParserHTMLGenerator.generateHTMLForEveryDocumented(doc)
+    code
+  }
+
+  /** Generates HTML from Documentation string.
+    */
+  def runOnPureDoc(comment: String): String = {
+    val doc  = DocParser.runMatched(comment)
+    val html = DocParserHTMLGenerator.generateHTMLPureDoc(doc)
+    html
+  }
+
+  /** Called if file doesn't contain docstrings, to let user know that they
+    * won't find anything at this page, and that it is not a bug.
+    */
+  def mapIfEmpty(doc: String): String = {
+    var tmp = doc
+    if (doc.replace("<div>", "").replace("</div>", "").length == 0) {
+      tmp =
+        "\n\n*Enso Reference Viewer.*\n\nNo documentation available for chosen source file."
+      tmp = runOnPureDoc(tmp).replace("style=\"font-size: 13px;\"", "")
+    }
+    tmp
+  }
+
+  /** Doc Parser may output file with many nested empty divs.
+    * This simple function will remove all unnecessary HTML tags.
+    */
+  def removeUnnecessaryDivs(doc: String): String = {
+    var tmp = doc
+    while (tmp.contains("<div></div>"))
+      tmp = tmp.replace("<div></div>", "")
+    tmp
+  }
+
+  /** Traverses through root directory, outputs list of all accessible files.
+    */
+  def traverse(root: File): LazyList[File] =
+    if (!root.exists) { LazyList.empty }
+    else {
+      LazyList.apply(root) ++ (root.listFiles match {
+        case null  => LazyList.empty
+        case files => files.view.flatMap(traverse)
+      })
+    }
+}

lib/scala/docs-generator/src/main/scala/org/enso/docs/generator/Main.scala

@@ -0,0 +1,220 @@
+package org.enso.docs.generator
+
+import java.io._
+import org.apache.commons.cli.{Option => CliOption, _}
+import scala.util.{Try, Using}
+import scala.io.Source
+import scalatags.Text.{all => HTML}
+import TreeOfCommonPrefixes._
+import DocsGenerator._
+import Constants._
+import HTML._
+
+/** The entry point for the documentation generator.
+  *
+  * The documentation generator is responsible for creating HTML documentation
+  * for the Enso standard library.
+  * It also generates JavaScript files containing react components for the
+  * [[https://enso.org/docs/reference reference website]].
+  */
+object Main {
+
+  /** Builds the [[Options]] object representing the CLI syntax.
+    *
+    * @return an [[Options]] object representing the CLI syntax
+    */
+  private def buildOptions = {
+    val help = CliOption
+      .builder("h")
+      .longOpt(HELP_OPTION)
+      .desc("Displays this message.")
+      .build
+    val input = CliOption.builder
+      .longOpt(INPUT_PATH_OPTION)
+      .numberOfArgs(1)
+      .argName("path")
+      .desc("Specifies working path.")
+      .build
+    val output = CliOption.builder
+      .hasArg(true)
+      .numberOfArgs(1)
+      .argName("directory")
+      .longOpt(OUTPUT_DIR_OPTION)
+      .desc("Specifies name of output directory.")
+      .build
+    val docsPath = CliOption.builder
+      .hasArg(true)
+      .numberOfArgs(1)
+      .argName("path")
+      .longOpt(DOCS_LIB_PATH_OPTION)
+      .desc("Specifies path of Docs Generator library.")
+      .build
+    val jsTemp = CliOption.builder
+      .hasArg(true)
+      .numberOfArgs(1)
+      .argName("file")
+      .longOpt(JS_TEMPLATE_OPTION)
+      .desc("Specifies name of file containing JS template.")
+      .build
+    val cssTemp = CliOption.builder
+      .hasArg(true)
+      .numberOfArgs(1)
+      .argName("file")
+      .longOpt(CSS_TEMPLATE_OPTION)
+      .desc("Specifies name of file containing CSS template.")
+      .build
+
+    val options = new Options
+    options
+      .addOption(help)
+      .addOption(input)
+      .addOption(output)
+      .addOption(docsPath)
+      .addOption(jsTemp)
+      .addOption(cssTemp)
+
+    options
+  }
+
+  /** Prints the help message to the standard output.
+    *
+    * @param options object representing the CLI syntax
+    */
+  private def printHelp(options: Options): Unit =
+    new HelpFormatter().printHelp("Docs Generator", options)
+
+  /** Terminates the process with a failure exit code. */
+  private def exitFail(): Nothing = sys.exit(1)
+
+  /** Terminates the process with a success exit code. */
+  private def exitSuccess(): Nothing = sys.exit(0)
+
+  /** Starting point. */
+  def main(args: Array[String]): Unit = {
+    val options = buildOptions
+    val parser  = new DefaultParser
+    val line = Try(parser.parse(options, args)).getOrElse {
+      printHelp(options)
+      exitFail()
+    }
+    if (line.hasOption(HELP_OPTION)) {
+      printHelp(options)
+      exitSuccess()
+    }
+    val path =
+      Option(line.getOptionValue(INPUT_PATH_OPTION)).getOrElse(SOURCE_PATH)
+    val outDir =
+      Option(line.getOptionValue(OUTPUT_DIR_OPTION)).getOrElse(OUTPUT_DIRECTORY)
+    val templateFilesPath = Option(line.getOptionValue(DOCS_LIB_PATH_OPTION))
+      .getOrElse(TEMPLATE_FILES_PATH)
+    val jsTempFileName = Option(line.getOptionValue(JS_TEMPLATE_OPTION))
+      .getOrElse(JS_TEMPLATE_NAME)
+    val cssFileName = Option(line.getOptionValue(CSS_TEMPLATE_OPTION))
+      .getOrElse(CSS_TREE_FILE_NAME)
+
+    generateAllDocs(
+      path,
+      templateFilesPath,
+      jsTempFileName,
+      cssFileName,
+      outDir
+    )
+  }
+
+  /** Traverses through directory generating docs from every .enso file found.
+    */
+  def generateAllDocs(
+    path: String,
+    templateFilesPath: String,
+    jsTempFileName: String,
+    cssFileName: String,
+    outDir: String
+  ): Unit = {
+    val allFiles = traverse(new File(path))
+      .filter(f => f.isFile && f.getName.endsWith(".enso"))
+    val allFileNames = allFiles.map(
+      _.getPath
+        .replace(path + "/", "")
+        .replace(".enso", "")
+    )
+    val allPrograms = allFiles
+      .map(f => Using(Source.fromFile(f, "UTF-8")) { _.mkString })
+      .toList
+    val allDocs = allPrograms
+      .map(s => run(s.getOrElse("")))
+      .map(mapIfEmpty)
+      .map(removeUnnecessaryDivs)
+    val treeNames =
+      groupByPrefix(allFileNames.toList, '/').filter(_.elems.nonEmpty)
+    val jsTemplate = new File(templateFilesPath + jsTempFileName)
+    val templateCode = Using(Source.fromFile(jsTemplate, "UTF-8")) {
+      _.mkString
+    }
+    val styleFile = new File(templateFilesPath + cssFileName)
+    val styleCode = Using(Source.fromFile(styleFile, "UTF-8")) { _.mkString }
+    val treeStyle = "<style jsx>{`" + styleCode.getOrElse("") + "`}</style>"
+    val allDocJSFiles = allFiles.map { x =>
+      val name = x.getPath
+        .replace(".enso", ".js")
+        .replace("Standard/src", outDir)
+        .replace("Main.js", "index.js")
+      val ending = name.split(outDir + "/").tail.head
+      name.replace(ending, ending.replace('/', '-'))
+    }
+    val dir = new File(allDocJSFiles.head.split(outDir).head + outDir + "/")
+    dir.mkdirs()
+    val zippedJS = allDocJSFiles.zip(allDocs)
+    zippedJS.foreach(d =>
+      createDocJSFile(d._1, d._2, outDir, treeStyle, templateCode, treeNames)
+    )
+  }
+
+  /** Takes a tuple of file path and documented HTML code, and generates JS doc
+    * file with react components for Enso website.
+    */
+  private def createDocJSFile(
+    path: String,
+    htmlCode: String,
+    outDir: String,
+    treeStyle: String,
+    templateCode: Try[String],
+    treeNames: List[Node]
+  ): Unit = {
+    val file = new File(path)
+    file.createNewFile()
+    val bw = new BufferedWriter(new FileWriter(file))
+    var treeCode =
+      "<div>" + treeStyle + HTML
+        .ul(treeNames.map(_.html()))
+        .render
+        .replace("class=", "className=") + "</div>"
+    if (path.contains("/index.js")) {
+      treeCode = treeCode.replace("a href=\"", "a href=\"reference/")
+    }
+    val partials = path
+      .split(outDir + "/")
+      .tail
+      .head
+      .replace(".js", "")
+      .split("-")
+    for (i <- 1 to partials.length) {
+      val id  = partials.take(i).mkString("-")
+      val beg = "<input type=\"checkbox\" id=\"" + id + "\" "
+      treeCode = treeCode.replace(beg, beg + "checked=\"True\"")
+    }
+    val docCopyBtnClass = "doc-copy-btn"
+    bw.write(
+      templateCode
+        .getOrElse("")
+        .replace(
+          "{/*PAGE*/}",
+          htmlCode
+            .replace(docCopyBtnClass + " flex", docCopyBtnClass + " none")
+            .replace("{", "&#123;")
+            .replace("}", "&#125;")
+        )
+        .replace("{/*BREADCRUMBS*/}", treeCode)
+    )
+    bw.close()
+  }
+}