Render the shortcodes as late as possible

This is needed to make shortcode users happy with the new multilanguage support,
but it will also solve many other related posts about "stuff not available in the shortcode".

We will have to revisit this re the handler chain at some point, but that will be easier
now as the integration test story has improved so much.

As part of this commit, the site-building tests in page_test.go is refreshed, they now
tests for all the rendering engines (when available), and all of them now uses the
same code-path as used in production.

Fixes #1229
Fixes #2323
Fixes ##1076

helpers/content.go

@@ -41,6 +41,8 @@ var SummaryLength = 70
 // SummaryDivider denotes where content summarization should end. The default is "<!--more-->".
 var SummaryDivider = []byte("<!--more-->")
 
+var summaryDividerAndNewLines = []byte("<!--more-->\n\n")
+
 // Blackfriday holds configuration values for Blackfriday rendering.
 type Blackfriday struct {
 	Smartypants                      bool
@@ -390,8 +392,21 @@ func WordCount(s string) map[string]int {
 }
 
 // RemoveSummaryDivider removes summary-divider <!--more--> from content.
+// TODO(bep) ml remove
 func RemoveSummaryDivider(content []byte) []byte {
-	return bytes.Replace(content, SummaryDivider, []byte(""), -1)
+	b := bytes.Replace(content, summaryDividerAndNewLines, []byte(""), 1)
+	if len(b) != len(content) {
+		return b
+	}
+	return bytes.Replace(content, SummaryDivider, []byte(""), 1)
+}
+
+func removeInternalSummaryDivider(content []byte) []byte {
+	b := bytes.Replace(content, summaryDividerAndNewLines, []byte(""), 1)
+	if len(b) != len(content) {
+		return b
+	}
+	return bytes.Replace(content, SummaryDivider, []byte(""), 1)
 }
 
 // TruncateWordsByRune truncates words by runes.

hugolib/handler_meta.go

@@ -75,8 +75,6 @@ func (mh *MetaHandle) Convert(i interface{}, s *Site, results HandleResults) {
 		}
 
 		results <- h.PageConvert(p, s.Tmpl)
-		p.setSummary()
-		p.analyzePage()
 	}
 }
 

hugolib/handler_page.go

@@ -14,10 +14,11 @@
 package hugolib
 
 import (
+	"bytes"
+
 	"github.com/spf13/hugo/helpers"
 	"github.com/spf13/hugo/source"
 	"github.com/spf13/hugo/tpl"
-	jww "github.com/spf13/jwalterweatherman"
 	"github.com/spf13/viper"
 )
 
@@ -67,18 +68,7 @@ type htmlHandler struct {
 func (h htmlHandler) Extensions() []string { return []string{"html", "htm"} }
 func (h htmlHandler) PageConvert(p *Page, t tpl.Template) HandledResult {
 	p.ProcessShortcodes(t)
-	var err error
-
-	if len(p.contentShortCodes) > 0 {
-		p.rawContent, err = replaceShortcodeTokens(p.rawContent, shortcodePlaceholderPrefix, p.contentShortCodes)
 
-		if err != nil {
-			jww.FATAL.Printf("Failed to replace short code tokens in %s:\n%s", p.BaseFileName(), err.Error())
-			return HandledResult{err: err}
-		}
-	}
-
-	p.Content = helpers.BytesToHTML(p.rawContent)
 	return HandledResult{err: nil}
 }
 
@@ -112,27 +102,22 @@ func (h mmarkHandler) PageConvert(p *Page, t tpl.Template) HandledResult {
 func commonConvert(p *Page, t tpl.Template) HandledResult {
 	p.ProcessShortcodes(t)
 
-	var err error
-
+	// TODO(bep) these page handlers need to be re-evaluated, as it is hard to
+	// process a page in isolation. See the new preRender func.
+	// TODO(bep) ml not so raw anymore, but do we need to keep it raw?
 	if viper.GetBool("EnableEmoji") {
 		p.rawContent = helpers.Emojify(p.rawContent)
 	}
 
-	renderedContent := p.renderContent(helpers.RemoveSummaryDivider(p.rawContent))
-
-	if len(p.contentShortCodes) > 0 {
-		renderedContent, err = replaceShortcodeTokens(renderedContent, shortcodePlaceholderPrefix, p.contentShortCodes)
-
-		if err != nil {
-			jww.FATAL.Printf("Failed to replace shortcode tokens in %s:\n%s", p.BaseFileName(), err.Error())
-			return HandledResult{err: err}
-		}
-	}
-
-	tmpContent, tmpTableOfContents := helpers.ExtractTOC(renderedContent)
+	// TODO(bep) ml we let the summary divider survive the rendering. Must check if
+	// it actually survives, replace it with something more robus, or maybe
+	// rethink this fragile concept.
+	//p.rawContent = p.renderContent(helpers.RemoveSummaryDivider(p.rawContent))
+	// We have to replace the <!--more--> with something that survives all the
+	// rendering engines.
+	// TODO(bep) inline replace
+	p.rawContent = bytes.Replace(p.rawContent, []byte(helpers.SummaryDivider), internalSummaryDivider, 1)
+	p.rawContent = p.renderContent(p.rawContent)
 
-	p.Content = helpers.BytesToHTML(tmpContent)
-	p.TableOfContents = helpers.BytesToHTML(tmpTableOfContents)
-	p.rendered = true
 	return HandledResult{err: nil}
 }

hugolib/hugo_sites.go

@@ -16,8 +16,11 @@ package hugolib
 import (
 	"errors"
 	"strings"
+	"sync"
 	"time"
 
+	"github.com/spf13/hugo/helpers"
+
 	"github.com/spf13/viper"
 
 	"github.com/fsnotify/fsnotify"
@@ -106,24 +109,27 @@ func (h HugoSites) Build(config BuildCfg) error {
 	}
 
 	for _, s := range h.Sites {
-
 		if err := s.PostProcess(); err != nil {
 			return err
 		}
+	}
+
+	if err := h.preRender(); err != nil {
+		return err
+	}
+
+	for _, s := range h.Sites {
 
 		if !config.skipRender {
 			if err := s.Render(); err != nil {
 				return err
 			}
 
+			if config.PrintStats {
+				s.Stats()
+			}
 		}
-
-		if config.PrintStats {
-			s.Stats()
-		}
-
 		// TODO(bep) ml lang in site.Info?
-		// TODO(bep) ml Page sorting?
 	}
 
 	if config.PrintStats {
@@ -153,22 +159,26 @@ func (h HugoSites) Rebuild(config BuildCfg, events ...fsnotify.Event) error {
 	// Assign pages to sites per translation.
 	h.setupTranslations(firstSite)
 
-	for _, s := range h.Sites {
-
-		if sourceChanged {
+	if sourceChanged {
+		for _, s := range h.Sites {
 			if err := s.PostProcess(); err != nil {
 				return err
 			}
 		}
+	}
 
-		if !config.skipRender {
+	if err := h.preRender(); err != nil {
+		return err
+	}
+
+	if !config.skipRender {
+		for _, s := range h.Sites {
 			if err := s.Render(); err != nil {
 				return err
 			}
-		}
-
-		if config.PrintStats {
-			s.Stats()
+			if config.PrintStats {
+				s.Stats()
+			}
 		}
 	}
 
@@ -219,6 +229,87 @@ func (s *HugoSites) setupTranslations(master *Site) {
 	}
 }
 
+// preRender performs build tasks that needs to be done as late as possible.
+// Shortcode handling is the main task in here.
+// TODO(bep) We need to look at the whole handler-chain construct witht he below in mind.
+func (h *HugoSites) preRender() error {
+	pageChan := make(chan *Page)
+
+	wg := &sync.WaitGroup{}
+
+	// We want all the pages, so just pick one.
+	s := h.Sites[0]
+
+	for i := 0; i < getGoMaxProcs()*4; i++ {
+		wg.Add(1)
+		go func(pages <-chan *Page, wg *sync.WaitGroup) {
+			defer wg.Done()
+			for p := range pages {
+				if err := handleShortcodes(p, s.Tmpl); err != nil {
+					jww.ERROR.Printf("Failed to handle shortcodes for page %s: %s", p.BaseFileName(), err)
+				}
+
+				if p.Markup == "markdown" {
+					tmpContent, tmpTableOfContents := helpers.ExtractTOC(p.rawContent)
+					p.TableOfContents = helpers.BytesToHTML(tmpTableOfContents)
+					p.rawContent = tmpContent
+				}
+
+				if p.Markup != "html" {
+
+					// Now we know enough to create a summary of the page and count some words
+					summaryContent, err := p.setUserDefinedSummaryIfProvided()
+
+					if err != nil {
+						jww.ERROR.Printf("Failed to set use defined summary: %s", err)
+					} else if summaryContent != nil {
+						p.rawContent = summaryContent.content
+					}
+
+					p.Content = helpers.BytesToHTML(p.rawContent)
+					p.rendered = true
+
+					if summaryContent == nil {
+						p.setAutoSummary()
+					}
+				}
+
+				//analyze for raw stats
+				p.analyzePage()
+			}
+		}(pageChan, wg)
+	}
+
+	for _, p := range s.AllPages {
+		pageChan <- p
+	}
+
+	close(pageChan)
+
+	wg.Wait()
+
+	return nil
+}
+
+func handleShortcodes(p *Page, t tpl.Template) error {
+	if len(p.contentShortCodes) > 0 {
+		jww.DEBUG.Printf("Replace %d shortcodes in %q", len(p.contentShortCodes), p.BaseFileName())
+		shortcodes, err := executeShortcodeFuncMap(p.contentShortCodes)
+
+		if err != nil {
+			return err
+		}
+
+		p.rawContent, err = replaceShortcodeTokens(p.rawContent, shortcodePlaceholderPrefix, shortcodes)
+
+		if err != nil {
+			jww.FATAL.Printf("Failed to replace short code tokens in %s:\n%s", p.BaseFileName(), err.Error())
+		}
+	}
+
+	return nil
+}
+
 func (s *Site) updateBuildStats(page *Page) {
 	if page.IsDraft() {
 		s.draftCount++

hugolib/hugo_sites_test.go

@@ -37,6 +37,11 @@ func testCommonResetState() {
 	viper.Set("PublishDir", "public")
 	viper.Set("RSSUri", "rss")
 
+	viper.Set("Taxonomies", map[string]interface{}{
+		"tag":      "tags",
+		"category": "categories",
+	})
+
 	if err := hugofs.Source().Mkdir("content", 0755); err != nil {
 		panic("Content folder creation failed.")
 	}