Merge pull request #285 from jroper/better-includes

Improved includes

core/src/main/scala/com/lightbend/paradox/ParadoxProcessor.scala

@@ -20,11 +20,13 @@ import com.lightbend.paradox.template.PageTemplate
 import com.lightbend.paradox.markdown._
 import com.lightbend.paradox.tree.Tree.{ Forest, Location }
 import java.io.{ File, FileOutputStream, OutputStreamWriter }
+import java.util
 
-import org.pegdown.ast.{ ClassyLinkNode, ExpLinkNode, RootNode }
+import org.pegdown.ast._
 import org.stringtemplate.v4.STErrorListener
 
 import scala.annotation.tailrec
+import scala.collection.JavaConverters._
 
 /**
  * Markdown site processor.
@@ -175,20 +177,63 @@ class ParadoxProcessor(reader: Reader = new Reader, writer: Writer = new Writer)
    * Parse markdown files (with paths) into a forest of linked pages.
    */
   def parsePages(mappings: Seq[(File, String)], convertPath: String => String, properties: Map[String, String]): Forest[Page] = {
-    Page.forest(parseMarkdown(mappings), convertPath, properties)
+    Page.forest(parseMarkdown(mappings, properties), convertPath, properties)
   }
 
   /**
    * Parse markdown files into pegdown AST.
    */
-  def parseMarkdown(mappings: Seq[(File, String)]): Seq[(File, String, RootNode, Map[String, String])] = {
+  def parseMarkdown(mappings: Seq[(File, String)], properties: Map[String, String]): Seq[(File, String, RootNode, Map[String, String])] = {
     mappings map {
       case (file, path) =>
         val frontin = Frontin(file)
-        (file, normalizePath(path), reader.read(frontin.body), frontin.header)
+        val root = parseAndProcessMarkdown(file, frontin.body, properties ++ frontin.header)
+        (file, normalizePath(path), root, frontin.header)
     }
   }
 
+  def parseAndProcessMarkdown(file: File, markdown: String, properties: Map[String, String]): RootNode = {
+    val root = reader.read(markdown)
+    processIncludes(file, root, properties)
+  }
+
+  private def processIncludes(file: File, root: RootNode, properties: Map[String, String]): RootNode = {
+    val newRoot = new RootNode
+    // This is a mutable list, and is expected to be mutated by anything that wishes to add children
+    val newChildren = newRoot.getChildren
+
+    root.getChildren.asScala.foreach {
+      case include: DirectiveNode if include.name == "include" =>
+        val labels = include.attributes.values("identifier").asScala
+        val source = include.source match {
+          case direct: DirectiveNode.Source.Direct => direct.value
+          case other                               => throw IncludeDirective.IncludeSourceException(other)
+        }
+        val includeFile = SourceDirective.resolveFile("include", source, file, properties)
+        val frontin = Frontin(includeFile)
+        val filterLabels = include.attributes.booleanValue(
+          "filterLabels",
+          properties.get("include.filterLabels").exists(_ == "true"))
+        val (text, snippetLang) = Snippet(includeFile, labels, filterLabels)
+        // I guess we could support multiple markup languages in future...
+        if (snippetLang != "md" && snippetLang != "markdown") {
+          throw IncludeDirective.IncludeFormatException(snippetLang)
+        }
+        val includedRoot = parseAndProcessMarkdown(includeFile, text, properties ++ frontin.header)
+        val includeNode = IncludeNode(includedRoot, includeFile, source)
+        includeNode.setStartIndex(include.getStartIndex)
+        includeNode.setEndIndex(include.getEndIndex)
+        newChildren.add(includeNode)
+
+      case other => newChildren.add(other)
+    }
+
+    newRoot.setReferences(root.getReferences)
+    newRoot.setAbbreviations(root.getAbbreviations)
+
+    newRoot
+  }
+
   /**
    * Normalize path to '/' separator.
    */

core/src/main/scala/com/lightbend/paradox/markdown/Directive.scala

@@ -20,7 +20,6 @@ import com.lightbend.paradox.tree.Tree.Location
 import java.io.{ File, FileNotFoundException }
 import java.util.Optional
 
-import com.lightbend.paradox.tree.Tree
 import org.pegdown.ast._
 import org.pegdown.ast.DirectiveNode.Format._
 import org.pegdown.plugins.ToHtmlSerializerPlugin
@@ -99,18 +98,7 @@ trait SourceDirective { this: Directive =>
   }
 
   protected def resolveFile(propPrefix: String, source: String, page: Page, variables: Map[String, String]): File =
-    source match {
-      case s if s startsWith "$" =>
-        val baseKey = s.drop(1).takeWhile(_ != '$')
-        val base = new File(PropertyUrl(s"$propPrefix.$baseKey.base_dir", variables.get).base.trim)
-        val effectiveBase = if (base.isAbsolute) base else new File(page.file.getParentFile, base.toString)
-        new File(effectiveBase, s.drop(baseKey.length + 2))
-      case s if s startsWith "/" =>
-        val base = new File(PropertyUrl(SnipDirective.buildBaseDir, variables.get).base.trim)
-        new File(base, s)
-      case s =>
-        new File(page.file.getParentFile, s)
-    }
+    SourceDirective.resolveFile(propPrefix, source, page.file, variables)
 
   private lazy val referenceMap: Map[String, ReferenceNode] = {
     val tempRoot = new RootNode
@@ -124,6 +112,22 @@ trait SourceDirective { this: Directive =>
   }
 }
 
+object SourceDirective {
+  def resolveFile(propPrefix: String, source: String, pageFile: File, variables: Map[String, String]): File =
+    source match {
+      case s if s startsWith "$" =>
+        val baseKey = s.drop(1).takeWhile(_ != '$')
+        val base = new File(PropertyUrl(s"$propPrefix.$baseKey.base_dir", variables.get).base.trim)
+        val effectiveBase = if (base.isAbsolute) base else new File(pageFile.getParentFile, base.toString)
+        new File(effectiveBase, s.drop(baseKey.length + 2))
+      case s if s startsWith "/" =>
+        val base = new File(PropertyUrl(SnipDirective.buildBaseDir, variables.get).base.trim)
+        new File(base, s)
+      case s =>
+        new File(pageFile.getParentFile, s)
+    }
+}
+
 // Default directives
 
 /**
@@ -360,8 +364,9 @@ case class SnipDirective(page: Page, variables: Map[String, String])
     try {
       val labels = node.attributes.values("identifier").asScala
       val source = resolvedSource(node, page)
+      val filterLabels = node.attributes.booleanValue("filterLabels", variables.get("snip.filterLabels").forall(_ == "true"))
       val file = resolveFile("snip", source, page, variables)
-      val (text, snippetLang) = Snippet(file, labels)
+      val (text, snippetLang) = Snippet(file, labels, filterLabels)
       val lang = Option(node.attributes.value("type")).getOrElse(snippetLang)
       val group = Option(node.attributes.value("group")).getOrElse("")
       val sourceUrl = if (variables.contains(GitHubResolver.baseUrl) && variables.getOrElse(SnipDirective.showGithubLinks, "false") == "true") {
@@ -412,7 +417,8 @@ case class FiddleDirective(page: Page, variables: Map[String, String])
 
       val source = resolvedSource(node, page)
       val file = resolveFile("fiddle", source, page, variables)
-      val (code, _) = Snippet(file, labels)
+      val filterLabels = node.attributes.booleanValue("filterLabels", variables.get("fiddle.filterLabels").forall(_ == "true"))
+      val (code, _) = Snippet(file, labels, filterLabels)
 
       printer.println.print(s"""
         <div data-scalafiddle $params>
@@ -444,7 +450,7 @@ object FiddleDirective {
  * Placeholder to insert a serialized table of contents, using the page and header trees.
  * Depth and whether to include pages or headers can be specified in directive attributes.
  */
-case class TocDirective(location: Location[Page]) extends LeafBlockDirective("toc") {
+case class TocDirective(location: Location[Page], includeIndexes: List[Int]) extends LeafBlockDirective("toc") {
   def render(node: DirectiveNode, visitor: Visitor, printer: Printer): Unit = {
     val classes = node.attributes.classesString
     val depth = node.attributes.intValue("depth", 6)
@@ -453,7 +459,7 @@ case class TocDirective(location: Location[Page]) extends LeafBlockDirective("to
     val ordered = node.attributes.booleanValue("ordered", false)
     val toc = new TableOfContents(pages, headers, ordered, depth)
     printer.println.print(s"""<div class="toc $classes">""")
-    toc.markdown(location, node.getStartIndex).accept(visitor)
+    toc.markdown(location, node.getStartIndex, includeIndexes).accept(visitor)
     printer.println.print("</div>")
   }
 }
@@ -697,29 +703,14 @@ object DependencyDirective {
   case class UndefinedVariable(name: String) extends RuntimeException(s"'$name' is not defined")
 }
 
-case class IncludeDirective(context: Writer.Context) extends LeafBlockDirective("include") with SourceDirective {
-
-  override def page: Page = context.location.tree.label
-  override def variables = context.properties
+case class IncludeDirective(page: Page, variables: Map[String, String]) extends LeafBlockDirective("include") with SourceDirective {
 
   override def render(node: DirectiveNode, visitor: Visitor, printer: Printer): Unit = {
-    val labels = node.attributes.values("identifier").asScala
-    val source = resolvedSource(node, page)
-    val file = resolveFile("include", source, page, context.properties)
-    val (text, snippetLang) = Snippet(file, labels)
-    // I guess we could support multiple markup languages in future...
-    if (snippetLang != "md" && snippetLang != "markdown") {
-      throw IncludeDirective.IncludeFormatException(snippetLang)
-    }
-    val includeNode = context.reader.read(text)
-
-    // This location has no forest around it... which probably means that things like toc and navigation can't
-    // be rendered inside snippets, which I'm ok with.
-    val newLocation = Location(Tree.leaf(Page.included(file, source, page, includeNode)), Nil, Nil, Nil)
-    printer.print(context.writer.writeContent(includeNode, context.copy(location = newLocation)))
+    throw new IllegalStateException("Include directive should have been handled in markdown preprocessing before render, but wasn't.")
   }
 }
 
 object IncludeDirective {
+  case class IncludeSourceException(source: DirectiveNode.Source) extends RuntimeException(s"Only explicit links are supported by the include directive, reference links are not: " + source)
   case class IncludeFormatException(format: String) extends RuntimeException(s"Don't know how to include '*.$format' content.")
 }

core/src/main/scala/com/lightbend/paradox/markdown/IncludeNode.scala

@@ -0,0 +1,47 @@
+/*
+ * Copyright © 2015 - 2017 Lightbend, Inc. <http://www.lightbend.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.lightbend.paradox.markdown
+
+import java.io.File
+import java.util
+
+import com.lightbend.paradox.markdown.Writer.Context
+import com.lightbend.paradox.tree.Tree
+import com.lightbend.paradox.tree.Tree.Location
+import org.pegdown.Printer
+import org.pegdown.ast.{ AbstractNode, Node, RootNode, Visitor }
+import org.pegdown.plugins.ToHtmlSerializerPlugin
+
+case class IncludeNode(included: RootNode, includedFrom: File, includedFromPath: String) extends AbstractNode {
+  override def accept(visitor: Visitor): Unit = visitor.visit(this)
+  override def getChildren: util.List[Node] = included.getChildren
+}
+
+class IncludeNodeSerializer(context: Context) extends ToHtmlSerializerPlugin {
+  override def visit(node: Node, visitor: Visitor, printer: Printer): Boolean = node match {
+    case include @ IncludeNode(included, includedFrom, includedFromPath) =>
+      // This location has no forest around it... which probably means that things like toc and navigation can't
+      // be rendered inside snippets, which I'm ok with.
+      val newLocation = Location(Tree.leaf(Page.included(includedFrom, includedFromPath,
+        context.location.tree.label, included)), context.location.lefts, context.location.rights, context.location.parents)
+      printer.print(context.writer.writeContent(included, context.copy(
+        location = newLocation,
+        includeIndexes = context.includeIndexes :+ include.getStartIndex)))
+      true
+    case _ => false
+  }
+}

core/src/main/scala/com/lightbend/paradox/markdown/Index.scala

@@ -28,7 +28,15 @@ import scala.collection.JavaConverters._
  */
 object Index {
 
-  case class Ref(level: Int, path: String, markdown: Node, group: Option[String])
+  /**
+   * @param level
+   * @param path
+   * @param markdown
+   * @param group
+   * @param includeIndexes If this header came from an included file, this has the index of the include file,
+   *                       starting from the top level page include, down to the deepest nesting.
+   */
+  case class Ref(level: Int, path: String, markdown: Node, group: Option[String], includeIndexes: List[Int])
 
   case class Page(file: File, path: String, markdown: RootNode, properties: Map[String, String], indices: Forest[Ref], headers: Forest[Ref])
 
@@ -46,25 +54,27 @@ object Index {
    * Create a tree of header refs from a parsed markdown page.
    */
   def headers(root: RootNode): Forest[Ref] = {
-    Tree.hierarchy(headerRefs(root, group = None))(Ordering[Int].on[Ref](_.level))
+    Tree.hierarchy(headerRefs(root, group = None, includeIndexes = Nil))(Ordering[Int].on[Ref](_.level))
   }
 
   /**
    * Extract refs from markdown headers.
    */
-  private def headerRefs(root: RootNode, group: Option[String]): List[Ref] = {
+  private def headerRefs(root: RootNode, group: Option[String], includeIndexes: List[Int]): List[Ref] = {
     root.getChildren.asScala.toList.flatMap {
       case header: HeaderNode =>
         header.getChildren.asScala.toList.flatMap {
-          case anchor: AnchorLinkSuperNode => List(Ref(header.getLevel, "#" + anchor.name, anchor.contents, group))
-          case anchor: AnchorLinkNode      => List(Ref(header.getLevel, "#" + anchor.getName, new TextNode(anchor.getText), group))
+          case anchor: AnchorLinkSuperNode => List(Ref(header.getLevel, "#" + anchor.name, anchor.contents, group, includeIndexes))
+          case anchor: AnchorLinkNode      => List(Ref(header.getLevel, "#" + anchor.getName, new TextNode(anchor.getText), group, includeIndexes))
           case _                           => Nil
         }
       case node: DirectiveNode if node.format == DirectiveNode.Format.ContainerBlock =>
         // TODO check whether my assumption that Container DirectiveNode's always contain RootNode's holds,
         // if so maybe move that cast to DirectiveNode
         val newGroup = node.attributes.classes().asScala.find(_.startsWith("group-")).map(_.substring("group-".size))
-        headerRefs(node.contentsNode.asInstanceOf[RootNode], newGroup)
+        headerRefs(node.contentsNode.asInstanceOf[RootNode], newGroup, includeIndexes)
+      case node @ IncludeNode(included, _, _) =>
+        headerRefs(included, group, includeIndexes :+ node.getStartIndex)
       case _ => Nil
     }
   }
@@ -109,7 +119,7 @@ object Index {
   @tailrec
   private def linkRef(node: Node, level: Int): Option[Ref] = {
     node match {
-      case link: ExpLinkNode => Some(Ref(level, link.url, link.getChildren.get(0), group = None))
+      case link: ExpLinkNode => Some(Ref(level, link.url, link.getChildren.get(0), group = None, Nil))
       case other => other.getChildren.asScala.toList match {
         // only check first children
         case first :: _ => linkRef(first, level)

core/src/main/scala/com/lightbend/paradox/markdown/Page.scala

@@ -36,7 +36,7 @@ sealed abstract class Linkable {
 /**
  * Header in a page, with anchor path and markdown nodes.
  */
-case class Header(path: String, label: Node, group: Option[String]) extends Linkable
+case class Header(path: String, label: Node, group: Option[String], includeIndexes: List[Int]) extends Linkable
 
 /**
  * Markdown page with target path, parsed markdown, and headers.
@@ -94,10 +94,10 @@ object Page {
     val targetPath = properties.convertToTarget(convertPath)(page.path)
     val rootSrcPage = Path.relativeRootPath(page.file, page.path)
     val (h1, subheaders) = page.headers match {
-      case h :: hs => (Header(h.label.path, h.label.markdown, h.label.group), h.children ++ hs)
-      case Nil     => (Header(targetPath, new SpecialTextNode(targetPath), None), Nil)
+      case h :: hs => (Header(h.label.path, h.label.markdown, h.label.group, h.label.includeIndexes), h.children ++ hs)
+      case Nil     => (Header(targetPath, new SpecialTextNode(targetPath), None, Nil), Nil)
     }
-    val headers = subheaders map (_ map (h => Header(h.path, h.markdown, h.group)))
+    val headers = subheaders map (_ map (h => Header(h.path, h.markdown, h.group, h.includeIndexes)))
     Page(page.file, targetPath, rootSrcPage, h1.label, h1, headers, page.markdown, h1.group, properties)
   }
 
@@ -149,8 +149,8 @@ object Page {
    */
   def included(file: File, includeFilePath: String, includedIn: Page, markdown: RootNode): Page = {
     val rootSrcPage = Path.relativeRootPath(file, includeFilePath)
-    val h1 = Header(includedIn.path, new SpecialTextNode(includedIn.path), None)
-    Page(file, includedIn.path, rootSrcPage, h1.label, h1, Nil, markdown, h1.group, includedIn.properties)
+    Page(file, includedIn.path, rootSrcPage, includedIn.h1.label, includedIn.h1, includedIn.headers, markdown,
+      includedIn.group, includedIn.properties)
   }
 }