From 630eecf240a18d90f193d54745d20be1cd821db7 Mon Sep 17 00:00:00 2001 From: Jens Halm <3116929+jenshalm@users.noreply.github.com> Date: Thu, 19 Jul 2018 23:29:32 +0100 Subject: [PATCH] unify handling of raw content extensions - #56 --- .../laika/api/config/OperationConfig.scala | 2 ++ .../laika/api/config/ParseConfigBuilder.scala | 8 ++++++++ .../scala/laika/parse/markdown/Markdown.scala | 16 ++++++---------- .../parse/markdown/html/HTMLRenderer.scala | 2 +- .../parse/markdown/html/VerbatimHTML.scala | 5 +++-- .../laika/parse/rst/ReStructuredText.scala | 18 ++++-------------- .../parse/rst/ext/RstExtensionSupport.scala | 1 + .../rst/ext/StandardBlockDirectives.scala | 2 +- .../parse/rst/ext/StandardTextRoles.scala | 2 +- .../parse/markdown/MarkdownToHTMLSpec.scala | 4 ++-- .../html/VerbatimHTMLRendererSpec.scala | 12 ++++++++---- .../parse/rst/ReStructuredTextToHTMLSpec.scala | 6 ++---- .../rst/ext/StandardBlockDirectivesSpec.scala | 2 +- .../parse/rst/ext/StandardTextRolesSpec.scala | 2 +- docs/using-laika/markup.md | 14 +++++++------- sbt/src/main/scala/laika/sbt/LaikaPlugin.scala | 15 +++++++-------- 16 files changed, 55 insertions(+), 56 deletions(-) diff --git a/core/src/main/scala/laika/api/config/OperationConfig.scala b/core/src/main/scala/laika/api/config/OperationConfig.scala index 94ec6ecbf..f399be3fa 100644 --- a/core/src/main/scala/laika/api/config/OperationConfig.scala +++ b/core/src/main/scala/laika/api/config/OperationConfig.scala @@ -75,6 +75,8 @@ case class OperationConfig (bundles: Seq[ExtensionBundle] = Nil, def forStrictMode: OperationConfig = copy(bundleFilter = bundleFilter.copy(strict = true)) + def forRawContent: OperationConfig = copy(bundleFilter = bundleFilter.copy(acceptRawContent = true)) + } object OperationConfig { diff --git a/core/src/main/scala/laika/api/config/ParseConfigBuilder.scala b/core/src/main/scala/laika/api/config/ParseConfigBuilder.scala index 28a7a30f7..710b56cd5 100644 --- a/core/src/main/scala/laika/api/config/ParseConfigBuilder.scala +++ b/core/src/main/scala/laika/api/config/ParseConfigBuilder.scala @@ -28,4 +28,12 @@ trait ParseConfigBuilder extends OperationConfigBuilder { */ def strict: ThisType = withConfig(config.forStrictMode) + /** Enables all extensions that process raw content embedded into the host + * markup language. + * These are disabled by default as Laika is designed to render to multiple + * output formats from a single input document. With raw content embedded + * the markup document is tied to a specific output format. + */ + def withRawContent: ThisType = withConfig(config.forRawContent) + } diff --git a/core/src/main/scala/laika/parse/markdown/Markdown.scala b/core/src/main/scala/laika/parse/markdown/Markdown.scala index cf2443d2e..5182749b7 100644 --- a/core/src/main/scala/laika/parse/markdown/Markdown.scala +++ b/core/src/main/scala/laika/parse/markdown/Markdown.scala @@ -22,6 +22,7 @@ import laika.factory.ParserFactory import laika.io.Input import laika.parse.core.combinator.Parsers import laika.parse.core.markup.DocumentParser +import laika.parse.markdown.html.VerbatimHTML import laika.tree.Documents.Document import laika.tree.Paths.Path @@ -41,7 +42,7 @@ import laika.tree.Paths.Path * It must be enabled explicitly: * * {{{ - * val document = Parse as Markdown using VerbatimHTML fromFile "hello.md" + * val document = Parse.as(Markdown).withRawContent.fromFile("hello.md") * }}} * * To switch off all custom extensions like directives, @@ -49,16 +50,17 @@ import laika.tree.Paths.Path * id generation for headers, you can run the parser in strict mode: * * {{{ - * Transform from Markdown.strict to HTML fromFile "hello.md" toFile "hello.html" + * Transform.from(Markdown).to(HTML).strict + * .fromFile("hello.md").toFile("hello.html") * }}} * * @author Jens Halm */ -class Markdown private () extends ParserFactory { +object Markdown extends ParserFactory { val fileSuffixes: Set[String] = Set("md","markdown") - val extensions = Seq() + val extensions = Seq(VerbatimHTML) /** The actual parser function, fully parsing the specified input and * returning a document tree. @@ -72,9 +74,3 @@ class Markdown private () extends ParserFactory { } } - -/** The default Markdown parser configuration, with verbatim HTML elements disabled. - * - * @author Jens Halm - */ -object Markdown extends Markdown() diff --git a/core/src/main/scala/laika/parse/markdown/html/HTMLRenderer.scala b/core/src/main/scala/laika/parse/markdown/html/HTMLRenderer.scala index 4ef6cd35a..da4286e05 100644 --- a/core/src/main/scala/laika/parse/markdown/html/HTMLRenderer.scala +++ b/core/src/main/scala/laika/parse/markdown/html/HTMLRenderer.scala @@ -30,7 +30,7 @@ import laika.tree.Elements.{RenderFunction, Text, TextContainer} * It must be applied explicitly as part of the `VerbatimHTML` bundle when enabling verbatim HTML: * * {{{ - * val transform = Transform from Markdown to HTML using VerbatimHTML + * val transform = Transform.from(Markdown).to(HTML).withRawContent * }}} * * @author Jens Halm diff --git a/core/src/main/scala/laika/parse/markdown/html/VerbatimHTML.scala b/core/src/main/scala/laika/parse/markdown/html/VerbatimHTML.scala index bad55504c..119aa190d 100644 --- a/core/src/main/scala/laika/parse/markdown/html/VerbatimHTML.scala +++ b/core/src/main/scala/laika/parse/markdown/html/VerbatimHTML.scala @@ -27,10 +27,10 @@ import laika.render.{HTML, HTMLWriter} * by this library as it aims to also support renderers for other formats than HTML, * this extension is disabled by default. * - * You can install it with the Transform API: + * You can enable it with the Transform API: * * {{{ - * val transform = Transform from Markdown to HTML using VerbatimHTML + * val transform = Transform.from(Markdown).to(HTML).withRawContent * }}} * * @author Jens Halm @@ -38,6 +38,7 @@ import laika.render.{HTML, HTMLWriter} object VerbatimHTML extends ExtensionBundle { override val useInStrictMode: Boolean = true + override val acceptRawContent: Boolean = true override def parserDefinitions: ParserDefinitionBuilders = ParserDefinitionBuilders( blockParsers = Seq(HTMLParsers.htmlBlockFragment), diff --git a/core/src/main/scala/laika/parse/rst/ReStructuredText.scala b/core/src/main/scala/laika/parse/rst/ReStructuredText.scala index 8335938ec..251ce4389 100644 --- a/core/src/main/scala/laika/parse/rst/ReStructuredText.scala +++ b/core/src/main/scala/laika/parse/rst/ReStructuredText.scala @@ -56,7 +56,7 @@ import laika.tree.Paths.Path * * @author Jens Halm */ -class ReStructuredText private (rawContent: Boolean = false) extends ParserFactory { self => +object ReStructuredText extends ParserFactory { self => val fileSuffixes: Set[String] = Set("rest","rst") @@ -71,13 +71,9 @@ class ReStructuredText private (rawContent: Boolean = false) extends ParserFacto } }, RstExtensionSupport, - StandardExtensions - ) ++ (if (rawContent) Seq(RawContentExtensions) else Nil) // TODO - move - - /** Adds the `raw` directive and text roles to the parser. - * These are disabled by default as they present a potential security risk. - */ - def withRawContent: ReStructuredText = new ReStructuredText(true) + StandardExtensions, + RawContentExtensions + ) /** The actual parser function, fully parsing the specified input and * returning a document tree. @@ -105,9 +101,3 @@ class ReStructuredText private (rawContent: Boolean = false) extends ParserFacto } } - -/** The default reStructuredText parser configuration. - * - * @author Jens Halm - */ -object ReStructuredText extends ReStructuredText(false) diff --git a/core/src/main/scala/laika/parse/rst/ext/RstExtensionSupport.scala b/core/src/main/scala/laika/parse/rst/ext/RstExtensionSupport.scala index be47e8866..c7253bc6c 100644 --- a/core/src/main/scala/laika/parse/rst/ext/RstExtensionSupport.scala +++ b/core/src/main/scala/laika/parse/rst/ext/RstExtensionSupport.scala @@ -203,6 +203,7 @@ object StandardExtensions extends RstExtensionRegistry { object RawContentExtensions extends RstExtensionRegistry { + override val acceptRawContent = true lazy val blockDirectives = Seq((new StandardBlockDirectives).rawDirective) lazy val spanDirectives = Seq() lazy val textRoles = Seq((new StandardTextRoles).rawTextRole) diff --git a/core/src/main/scala/laika/parse/rst/ext/StandardBlockDirectives.scala b/core/src/main/scala/laika/parse/rst/ext/StandardBlockDirectives.scala index 50c9ee5f6..47a21c3b4 100644 --- a/core/src/main/scala/laika/parse/rst/ext/StandardBlockDirectives.scala +++ b/core/src/main/scala/laika/parse/rst/ext/StandardBlockDirectives.scala @@ -286,7 +286,7 @@ class StandardBlockDirectives { /** The raw directive, which is not enabled by default, * see [[http://docutils.sourceforge.net/docs/ref/rst/directives.html#raw-data-pass-through]] for details. - * It can be enabled with `ReStructuredText.withRawContent`. + * It can be enabled with `Transform.from(ReStructuredText).to(HTML).withRawContent`. */ lazy val rawDirective: Directive[Block] = BlockDirective("raw") { (argument(withWS = true) ~ content(Right(_))) { (formats, content) => diff --git a/core/src/main/scala/laika/parse/rst/ext/StandardTextRoles.scala b/core/src/main/scala/laika/parse/rst/ext/StandardTextRoles.scala index 8ee35319d..6371abfdb 100644 --- a/core/src/main/scala/laika/parse/rst/ext/StandardTextRoles.scala +++ b/core/src/main/scala/laika/parse/rst/ext/StandardTextRoles.scala @@ -124,7 +124,7 @@ class StandardTextRoles { /** The raw text role, which is not enabled by default, * see [[http://docutils.sourceforge.net/docs/ref/rst/roles.html#raw]] for details. - * It can be enabled with `ReStructuredText.withRawContent`. + * It can be enabled with `Transform.from(ReStructuredText).to(HTML).withRawContent`. */ lazy val rawTextRole: TextRole = TextRole("raw", (Nil:List[String],NoOpt:Options)) { diff --git a/core/src/test/scala/laika/parse/markdown/MarkdownToHTMLSpec.scala b/core/src/test/scala/laika/parse/markdown/MarkdownToHTMLSpec.scala index 5afba5211..4736d14d0 100644 --- a/core/src/test/scala/laika/parse/markdown/MarkdownToHTMLSpec.scala +++ b/core/src/test/scala/laika/parse/markdown/MarkdownToHTMLSpec.scala @@ -60,7 +60,7 @@ class MarkdownToHTMLSpec extends FlatSpec // TODO - remove once strict mode is handled properly object StrictMarkdown extends ParserFactory { val fileSuffixes: Set[String] = Set("md","markdown") - val extensions = Seq() + val extensions = Seq(VerbatimHTML) def newParser (parserExtensions: ParserDefinitionBuilders): Input => Document = { val rootParser = new RootParser(parserExtensions.blockParsers, parserExtensions.spanParsers, isStrict = true) val configHeaderParsers = parserExtensions.configHeaderParsers :+ { _:Path => Parsers.success(Right(ConfigFactory.empty)) } @@ -71,7 +71,7 @@ class MarkdownToHTMLSpec extends FlatSpec def transformAndCompare (name: String): Unit = { val path = classPathResource("/markdownTestSuite") + "/" + name - val actual = (Transform from StrictMarkdown to HTML using VerbatimHTML rendering { out => { + val actual = ((Transform from StrictMarkdown to HTML).withRawContent rendering { out => { case QuotedBlock(content,_,_) => out << "
" <<|> content <<| "
" // Markdown always writes p tags inside blockquotes }}).strict fromFile (path + ".md") toString val expected = readFile(path + ".html") diff --git a/core/src/test/scala/laika/parse/markdown/html/VerbatimHTMLRendererSpec.scala b/core/src/test/scala/laika/parse/markdown/html/VerbatimHTMLRendererSpec.scala index 3b6d6678d..74693631c 100644 --- a/core/src/test/scala/laika/parse/markdown/html/VerbatimHTMLRendererSpec.scala +++ b/core/src/test/scala/laika/parse/markdown/html/VerbatimHTMLRendererSpec.scala @@ -19,8 +19,8 @@ package laika.parse.markdown.html import org.scalatest.FlatSpec import org.scalatest.junit.JUnitRunner import org.scalatest.Matchers - -import laika.api.Render +import laika.api.{Parse, Render} +import laika.parse.markdown.Markdown import laika.parse.markdown.html.HTMLElements._ import laika.render.HTML import laika.tree.Elements.Element @@ -31,9 +31,13 @@ class VerbatimHTMLRendererSpec extends FlatSpec with Matchers with ModelBuilder with HTMLModelBuilder { - + + val renderer = { + val parser = Parse.as(Markdown).withRawContent + Render.as(HTML).withConfig(parser.config) + } - def render (elem: Element): String = Render as HTML using VerbatimHTML from elem toString + def render (elem: Element): String = renderer from elem toString "The Verbatim HTML renderer" should "render an HTML character reference unescaped" in { diff --git a/core/src/test/scala/laika/parse/rst/ReStructuredTextToHTMLSpec.scala b/core/src/test/scala/laika/parse/rst/ReStructuredTextToHTMLSpec.scala index c611cca4b..f7b6b8733 100644 --- a/core/src/test/scala/laika/parse/rst/ReStructuredTextToHTMLSpec.scala +++ b/core/src/test/scala/laika/parse/rst/ReStructuredTextToHTMLSpec.scala @@ -16,15 +16,13 @@ package laika.parse.rst -import org.scalatest.FlatSpec -import org.scalatest.Matchers import laika.api.Transform -import laika.parse.markdown.html.VerbatimHTML import laika.render.HTML import laika.transform.helper.FileTransformerUtil import laika.tree.Elements._ +import org.scalatest.{FlatSpec, Matchers} + import scala.io.Codec -import laika.io.Input /** * @author Jens Halm diff --git a/core/src/test/scala/laika/parse/rst/ext/StandardBlockDirectivesSpec.scala b/core/src/test/scala/laika/parse/rst/ext/StandardBlockDirectivesSpec.scala index 74f7ffd93..ae9f77c47 100644 --- a/core/src/test/scala/laika/parse/rst/ext/StandardBlockDirectivesSpec.scala +++ b/core/src/test/scala/laika/parse/rst/ext/StandardBlockDirectivesSpec.scala @@ -673,7 +673,7 @@ class StandardBlockDirectivesSpec extends FlatSpec | | some more""".stripMargin val result = root (RawContent(List("format"), "some input\n\nsome more")) - (Parse as ReStructuredText.withRawContent fromString input).content should be (result) + Parse.as(ReStructuredText).withRawContent.fromString(input).content should be (result) } diff --git a/core/src/test/scala/laika/parse/rst/ext/StandardTextRolesSpec.scala b/core/src/test/scala/laika/parse/rst/ext/StandardTextRolesSpec.scala index 362027991..e9eb8684d 100644 --- a/core/src/test/scala/laika/parse/rst/ext/StandardTextRolesSpec.scala +++ b/core/src/test/scala/laika/parse/rst/ext/StandardTextRolesSpec.scala @@ -179,7 +179,7 @@ class StandardTextRolesSpec extends FlatSpec | |some :foo:`text`""".stripMargin val result = root(p(txt("some "), RawContent(List("AML","BML","CML"), "text", Styles("foo")))) - (Parse as ReStructuredText.withRawContent fromString input).content should be (result) + Parse.as(ReStructuredText).withRawContent.fromString(input).content should be (result) } it should "be disabled by default" in { diff --git a/docs/using-laika/markup.md b/docs/using-laika/markup.md index ed0422563..d1737fc7c 100644 --- a/docs/using-laika/markup.md +++ b/docs/using-laika/markup.md @@ -27,8 +27,8 @@ Laika supports several convenient features for processing groups of documents. These are built as extensions to both Markdown and reStructuredText parsers. They can be switched off when you run these two parsers in strict mode: - Transform from Markdown.strict to - HTML fromFile "hello.md" toFile "hello.html" + Transform.from(Markdown).to(HTML).strict + .fromFile("hello.md").toFile("hello.html") The extensions are documented in their respective section linked to from the list below: @@ -104,8 +104,8 @@ Finally there is one major difference to standard Markdown: the parsing of verba is not enabled by default, but it can be switched on if required. When using this feature -you need to be aware of the fact that it ties your markup files to HTML output. Future -versions of Laika are supposed to support formats like PDF, epub and DocBook, and markup +you need to be aware of the fact that it ties your markup files to HTML output. Laika +supports multiple output formats like HTML, PDF, XSL-FO and in the future epub, and markup files containing raw HTML could not be used for those. When the markup originates from user input in a web application, it would not be safe @@ -115,11 +115,11 @@ customization hooks like [Document Tree Rewriting] or [Customizing Renderers]. To enable verbatim HTML elements you have to change this standard expression: - Transform from Markdown to HTML + Transform.from(Markdown).to(HTML) to - Transform from Markdown to HTML using VerbatimHTML + Transform.from(Markdown).to(HTML).withRawContent This installs both, the required parser and renderer extensions. @@ -241,7 +241,7 @@ seamed too exotic to warrant inclusion in Laika. Finally some of the defaults for these extensions can be changed through the API: - ReStructuredText.withRawContent + val transformer = Transform.from(ReStructuredText).to(HTML).withRawContent enables both the `raw` directive and the `raw` text role. They are disabled by default as they present a potential security risk. diff --git a/sbt/src/main/scala/laika/sbt/LaikaPlugin.scala b/sbt/src/main/scala/laika/sbt/LaikaPlugin.scala index 26f903ddf..f37af1046 100644 --- a/sbt/src/main/scala/laika/sbt/LaikaPlugin.scala +++ b/sbt/src/main/scala/laika/sbt/LaikaPlugin.scala @@ -17,15 +17,14 @@ package laika.sbt import laika.api._ -import laika.api.ext.ExtensionBundle.LaikaDefaults -import laika.directive.Directives._ import laika.directive.DirectiveRegistry +import laika.directive.Directives._ import laika.io.Input.LazyFileInput import laika.io.{DocumentType, Input, InputTree, OutputTree} import laika.parse.markdown.Markdown -import laika.parse.markdown.html.{HTMLRenderer, VerbatimHTML} -import laika.parse.rst.ext.{Directives, RstExtensionRegistry} +import laika.parse.markdown.html.HTMLRenderer import laika.parse.rst.ext.TextRoles.TextRole +import laika.parse.rst.ext.{Directives, RstExtensionRegistry} import laika.parse.rst.{ExtendedHTML, ReStructuredText} import laika.render._ import laika.rewrite.{DocumentCursor, RewriteRules} @@ -124,8 +123,8 @@ object LaikaPlugin extends AutoPlugin { } - import autoImport._ import Tasks._ + import autoImport._ override def projectSettings: Seq[Setting[_]] = Seq( sourceDirectories in Laika := Seq(sourceDirectory.value / "docs"), @@ -165,9 +164,9 @@ object LaikaPlugin extends AutoPlugin { val spanDirectives = rstSpanDirectives.value val textRoles = rstTextRoles.value } - val rst = if (laikaRawContent.value) ReStructuredText.withRawContent else ReStructuredText - val parser = Parse.as(Markdown).or(rst).withoutRewrite.using(directives, rstExtensions) - val pWithRaw = if (laikaRawContent.value) parser using VerbatimHTML else parser + val parser = Parse.as(Markdown).or(ReStructuredText) + .withoutRewrite.using(directives, rstExtensions) + val pWithRaw = if (laikaRawContent.value) parser.withRawContent else parser val pWithPar = if (laikaParallel.value) pWithRaw.inParallel else pWithRaw if (laikaStrict.value) pWithPar.strict else pWithPar },