Add Pandoc support, refactor external helpers

Recognize the Pandoc format under the file extension .pandoc or .pdc,
and shell out to pandoc as an external helper to format Pandoc content.

Add a configuration option, externalHelperArguments, that can be used to
override the additional arguments passed to external helpers.

Refactor out repeated code with external helpers. Change the error
output formatting. I did not see any of the external helpers print the
string "<input>" to represent stdin as a file; just prepending the file
name to error output is more general and doesn't sacrifice that much in
terms of readability.

docs/content/content-management/formats.md

@@ -6,7 +6,7 @@ date: 2017-01-10
publishdate: 2017-01-10
lastmod: 2017-04-06
categories: [content management]
keywords: [markdown,asciidoc,mmark,content format]
keywords: [markdown,asciidoc,mmark,pandoc,content format]
menu:
  docs:
    parent: "content-management"
@@ -195,12 +195,33 @@ With this setup, everything is in place for a natural usage of MathJax on pages

## Additional Formats Through External Helpers

Hugo has new concept called _external helpers_. It means that you can write your content using [Asciidoc][ascii], [reStructuredText][rest]. If you have files with associated extensions, Hugo will call external commands to generate the content. ([See the Hugo source code for external helpers][helperssource].)
Hugo has a new concept called _external helpers_. It means that you can write your content using [Asciidoc][ascii], [reStructuredText][rest], or [pandoc]. If you have files with associated extensions, Hugo will call external commands to generate the content. ([See the Hugo source code for external helpers][helperssource].)

For example, for Asciidoc files, Hugo will try to call the `asciidoctor` or `asciidoc` command. This means that you will have to install the associated tool on your machine to be able to use these formats. ([See the Asciidoctor docs for installation instructions](http://asciidoctor.org/docs/install-toolchain/)).

To use these formats, just use the standard extension and the front matter exactly as you would do with natively supported `.md` files.

Hugo attempts to pass reasonable default arguments to these external helpers by default:

- `asciidoc`: `--no-header-footer --safe -`
- `asciidoctor`: `--no-header-footer --safe --trace -`
- `rst2html`: `--leave-comments --initial-header-level=2`
- `pandoc`: (none)

You can override these arguments by adding entries under the `externalHelperArguments` variable in your [site configuration][config]. For example, if you wanted to use `pandoc` to render your markup with the flag `--strip-comments`, you might put the following in `config.toml`:

```
[externalHelperArguments]
  pandoc: --strip-comments
```

Or in `config.yaml`:

```
externalHelperArguments:
  pandoc: --strip-comments
```

{{% warning "Performance of External Helpers" %}}
Because additional formats are external commands generation performance will rely heavily on the performance of the external tool you are using. As this feature is still in its infancy, feedback is welcome.
{{% /warning %}}
@@ -235,6 +256,7 @@ Markdown syntax is simple enough to learn in a single sitting. The following are
[mmark]: https://github.com/miekg/mmark
[mmarkgh]: https://github.com/miekg/mmark/wiki/Syntax
[org]: http://orgmode.org/
[pandoc]: http://www.pandoc.org/
[Pygments]: http://pygments.org/
[rest]: http://docutils.sourceforge.net/rst.html
[sc]: /content-management/shortcodes/

helpers/content.go

@@ -46,6 +46,10 @@ type ContentSpec struct {
	footnoteReturnLinkContents string
	// SummaryLength is the length of the summary that Hugo extracts from a content.
	summaryLength int
	// map of format to args that should be passed to the external helper,
	// as either a string parsed as a list of whitespace-separated tokens
	// or as list of strings
	externalHelperArguments map[string]interface{}

	Highlight            func(code, lang, optsStr string) (string, error)
	defatultPygmentsOpts map[string]string
@@ -61,6 +65,7 @@ func NewContentSpec(cfg config.Provider) (*ContentSpec, error) {
		footnoteAnchorPrefix:       cfg.GetString("footnoteAnchorPrefix"),
		footnoteReturnLinkContents: cfg.GetString("footnoteReturnLinkContents"),
		summaryLength:              cfg.GetInt("summaryLength"),
		externalHelperArguments:    cfg.GetStringMap("externalHelperArguments"),

		cfg: cfg,
	}
@@ -439,6 +444,23 @@ type RenderingContext struct {
	Cfg          config.Provider
}

func (c ContentSpec) getExternalHelperArguments(fmt string) []string {
	args := c.externalHelperArguments[fmt]
	if args == nil {
		return nil
	}
	switch args := args.(type) {
	case []string:
		return args
	case string:
		return strings.Fields(args)
	default:
		jww.ERROR.Printf("Could not understand external helper arguments %v", args)
		jww.ERROR.Println("Falling back to default arguments")
		return nil
	}
}

// RenderBytes renders a []byte.
func (c ContentSpec) RenderBytes(ctx *RenderingContext) []byte {
	switch ctx.PageFmt {
@@ -447,13 +469,15 @@ func (c ContentSpec) RenderBytes(ctx *RenderingContext) []byte {
	case "markdown":
		return c.markdownRender(ctx)
	case "asciidoc":
		return getAsciidocContent(ctx)
		return getAsciidocContent(ctx, c.getExternalHelperArguments("asciidoc"))
	case "mmark":
		return c.mmarkRender(ctx)
	case "rst":
		return getRstContent(ctx)
		return getRstContent(ctx, c.getExternalHelperArguments("rst"))
	case "org":
		return orgRender(ctx, c)
	case "pandoc":
		return getPandocContent(ctx, c.getExternalHelperArguments("pandoc"))
	}
}

@@ -578,11 +602,6 @@ func getAsciidocExecPath() string {
	return path
}

// HasAsciidoc returns whether Asciidoc is installed on this computer.
func HasAsciidoc() bool {
	return getAsciidocExecPath() != ""
}

func getAsciidoctorExecPath() string {
	path, err := exec.LookPath("asciidoctor")
	if err != nil {
@@ -591,56 +610,38 @@ func getAsciidoctorExecPath() string {
	return path
}

// HasAsciidoctor returns whether Asciidoctor is installed on this computer.
func HasAsciidoctor() bool {
	return getAsciidoctorExecPath() != ""
// HasAsciidoc returns whether Asciidoc or Asciidoctor is installed on this computer.
func HasAsciidoc() bool {
	return (getAsciidoctorExecPath() != "" ||
		getAsciidocExecPath() != "")
}

// getAsciidocContent calls asciidoctor or asciidoc as an external helper
// to convert AsciiDoc content to HTML.
func getAsciidocContent(ctx *RenderingContext) []byte {
	content := ctx.Content
	cleanContent := bytes.Replace(content, SummaryDivider, []byte(""), 1)

func getAsciidocContent(ctx *RenderingContext, args []string) []byte {
	var isAsciidoctor bool
	path := getAsciidoctorExecPath()
	if path == "" {
		path = getAsciidocExecPath()
		if path == "" {
			jww.ERROR.Println("asciidoctor / asciidoc not found in $PATH: Please install.\n",
				"                 Leaving AsciiDoc content unrendered.")
			return content
			return ctx.Content
		}
	} else {
		isAsciidoctor = true
	}

	jww.INFO.Println("Rendering", ctx.DocumentName, "with", path, "...")
	args := []string{"--no-header-footer", "--safe"}
	if isAsciidoctor {
		// asciidoctor-specific arg to show stack traces on errors
		args = append(args, "--trace")
	}
	args = append(args, "-")
	cmd := exec.Command(path, args...)
	cmd.Stdin = bytes.NewReader(cleanContent)
	var out, cmderr bytes.Buffer
	cmd.Stdout = &out
	cmd.Stderr = &cmderr
	err := cmd.Run()
	// asciidoctor has exit code 0 even if there are errors in stderr
	// -> log stderr output regardless of state of err
	for _, item := range strings.Split(string(cmderr.Bytes()), "\n") {
		item := strings.TrimSpace(item)
		if item != "" {
			jww.ERROR.Println(strings.Replace(item, "<stdin>", ctx.DocumentName, 1))
	if args == nil {
		args = []string{"--no-header-footer", "--safe"}
		if isAsciidoctor {
			// asciidoctor-specific arg to show stack traces on errors
			args = append(args, "--trace")
		}
		args = append(args, "-")
	}
	if err != nil {
		jww.ERROR.Printf("%s rendering %s: %v", path, ctx.DocumentName, err)
	}

	return normalizeExternalHelperLineFeeds(out.Bytes())
	return externallyRenderContent(ctx, path, args)
}

// HasRst returns whether rst2html is installed on this computer.
@@ -672,41 +673,23 @@ func getPythonExecPath() string {

// getRstContent calls the Python script rst2html as an external helper
// to convert reStructuredText content to HTML.
func getRstContent(ctx *RenderingContext) []byte {
	content := ctx.Content
	cleanContent := bytes.Replace(content, SummaryDivider, []byte(""), 1)

func getRstContent(ctx *RenderingContext, args []string) []byte {
	python := getPythonExecPath()
	path := getRstExecPath()

	if path == "" {
		jww.ERROR.Println("rst2html / rst2html.py not found in $PATH: Please install.\n",
			"                 Leaving reStructuredText content unrendered.")
		return content
		return ctx.Content

	}

	jww.INFO.Println("Rendering", ctx.DocumentName, "with", path, "...")
	cmd := exec.Command(python, path, "--leave-comments", "--initial-header-level=2")
	cmd.Stdin = bytes.NewReader(cleanContent)
	var out, cmderr bytes.Buffer
	cmd.Stdout = &out
	cmd.Stderr = &cmderr
	err := cmd.Run()
	// By default rst2html exits w/ non-zero exit code only if severe, i.e.
	// halting errors occurred. -> log stderr output regardless of state of err
	for _, item := range strings.Split(string(cmderr.Bytes()), "\n") {
		item := strings.TrimSpace(item)
		if item != "" {
			jww.ERROR.Println(strings.Replace(item, "<stdin>", ctx.DocumentName, 1))
		}
	}
	if err != nil {
		jww.ERROR.Printf("%s rendering %s: %v", path, ctx.DocumentName, err)
	if args == nil {
		args = []string{path, "--leave-comments", "--initial-header-level=2"}
	} else {
		args = append([]string{path}, args...)
	}

	result := normalizeExternalHelperLineFeeds(out.Bytes())

	result := externallyRenderContent(ctx, python, args)
	// TODO(bep) check if rst2html has a body only option.
	bodyStart := bytes.Index(result, []byte("<body>\n"))
	if bodyStart < 0 {
@@ -724,9 +707,48 @@ func getRstContent(ctx *RenderingContext) []byte {
	return result[bodyStart+7 : bodyEnd]
}

// getPandocContent calls pandoc as an external helper to convert pandoc markdown to HTML.
func getPandocContent(ctx *RenderingContext, args []string) []byte {
	content := ctx.Content
	path, err := exec.LookPath("pandoc")
	if err != nil {
		jww.ERROR.Println("pandoc not found in $PATH: Please install.\n",
			"                 Leaving pandoc content unrendered.")
		return content
	}
	// the default args we want to pass to pandoc are already empty, so we
	// don't need to check for args == nil
	return externallyRenderContent(ctx, path, args)
}

func orgRender(ctx *RenderingContext, c ContentSpec) []byte {
	content := ctx.Content
	cleanContent := bytes.Replace(content, []byte("# more"), []byte(""), 1)
	return goorgeous.Org(cleanContent,
		c.getHTMLRenderer(blackfriday.HTML_TOC, ctx))
}

func externallyRenderContent(ctx *RenderingContext, path string, args []string) []byte {
	content := ctx.Content
	cleanContent := bytes.Replace(content, SummaryDivider, []byte(""), 1)

	cmd := exec.Command(path, args...)
	cmd.Stdin = bytes.NewReader(cleanContent)
	var out, cmderr bytes.Buffer
	cmd.Stdout = &out
	cmd.Stderr = &cmderr
	err := cmd.Run()
	// Most external helpers exit w/ non-zero exit code only if severe, i.e.
	// halting errors occurred. -> log stderr output regardless of state of err
	for _, item := range strings.Split(string(cmderr.Bytes()), "\n") {
		item := strings.TrimSpace(item)
		if item != "" {
			jww.ERROR.Printf("%s: %s", ctx.DocumentName, item)
		}
	}
	if err != nil {
		jww.ERROR.Printf("%s rendering %s: %v", path, ctx.DocumentName, err)
	}

	return normalizeExternalHelperLineFeeds(out.Bytes())
}

helpers/general.go

@@ -78,6 +78,8 @@ func GuessType(in string) string {
		return "mmark"
	case "rst":
		return "rst"
	case "pandoc", "pdc":
		return "pandoc"
	case "html", "htm":
		return "html"
	case "org":

helpers/general_test.go

@@ -34,6 +34,8 @@ func TestGuessType(t *testing.T) {
		{"adoc", "asciidoc"},
		{"ad", "asciidoc"},
		{"rst", "rst"},
		{"pandoc", "pandoc"},
		{"pdc", "pandoc"},
		{"mmark", "mmark"},
		{"html", "html"},
		{"htm", "html"},

hugolib/handler_page.go

@@ -25,6 +25,7 @@ func init() {
	RegisterHandler(new(htmlHandler))
	RegisterHandler(new(asciidocHandler))
	RegisterHandler(new(rstHandler))
	RegisterHandler(new(pandocHandler))
	RegisterHandler(new(mmarkHandler))
	RegisterHandler(new(orgHandler))
}
@@ -104,6 +105,15 @@ func (h rstHandler) PageConvert(p *Page) HandledResult {
	return commonConvert(p)
}

type pandocHandler struct {
	basicPageHandler
}

func (h pandocHandler) Extensions() []string { return []string{"pandoc", "pdc"} }
func (h pandocHandler) PageConvert(p *Page) HandledResult {
	return commonConvert(p)
}

type mmarkHandler struct {
	basicPageHandler
}

hugolib/page_test.go

@@ -561,7 +561,7 @@ func testAllMarkdownEnginesForPages(t *testing.T,
	}{
		{"md", func() bool { return true }},
		{"mmark", func() bool { return true }},
		{"ad", func() bool { return helpers.HasAsciidoctor() || helpers.HasAsciidoc() }},
		{"ad", func() bool { return helpers.HasAsciidoc() }},
		// TODO(bep) figure a way to include this without too much work.{"html", func() bool { return true }},
		{"rst", func() bool { return helpers.HasRst() }},
	}

hugolib/shortcode_test.go

@@ -565,7 +565,7 @@ tags:
	th := testHelper{s.Cfg, s.Fs, t}

	for _, test := range tests {
		if strings.HasSuffix(test.contentPath, ".ad") && !helpers.HasAsciidoctor() && !helpers.HasAsciidoc() {
		if strings.HasSuffix(test.contentPath, ".ad") && !helpers.HasAsciidoc() {
			fmt.Println("Skip Asciidoc test case as no Asciidoc present.")
			continue
		} else if strings.HasSuffix(test.contentPath, ".rst") && !helpers.HasRst() {