Make .Content (almost) always available in shortcodes

This resolves some surprising behaviour when reading other pages' content from shortcodes. Before this commit, that behaviour has been undefined. Note that this has never been an issue from regular templates.

It will still not be possible to get **the current shortcode's  page's rendered content**. That would have impressed Einstein.

The new and well defined rules are:

* `.Page.Content` from a shortcode will be empty. The related `.Page.Truncated` `.Page.Summary`, `.Page.WordCount`, `.Page.ReadingTime`, `.Page.Plain` and `.Page.PlainWords` will also have empty values.
* For _other pages_ (retrieved via `.Page.Site.GetPage`, `.Site.Pages` etc.) the `.Content` is there to use as you please as long as you don't have infinite content recursion in your shortcode/content setup. See below.
* `.Page.TableOfContents` is good to go (but does not support shortcodes in headlines; this is unchanged)

If you get into a situation of infinite recursion, the `.Content` will be empty. Run `hugo -v` for more information.

Fixes gohugoio#4632
Fixes gohugoio#4653
Fixes gohugoio#4655

deps/deps.go

@@ -4,6 +4,7 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
+	"time"
 
 	"github.com/gohugoio/hugo/config"
 	"github.com/gohugoio/hugo/helpers"
@@ -54,6 +55,9 @@ type Deps struct {
 	translationProvider ResourceProvider
 
 	Metrics metrics.Provider
+
+	// Timeout is configurable in site config.
+	Timeout time.Duration
 }
 
 // ResourceProvider is used to create and refresh, and clone resources needed.
@@ -128,6 +132,11 @@ func New(cfg DepsCfg) (*Deps, error) {
 
 	sp := source.NewSourceSpec(ps, fs.Source)
 
+	timeoutms := cfg.Language.GetInt("timeout")
+	if timeoutms <= 0 {
+		timeoutms = 3000
+	}
+
 	d := &Deps{
 		Fs:                  fs,
 		Log:                 logger,
@@ -139,6 +148,7 @@ func New(cfg DepsCfg) (*Deps, error) {
 		SourceSpec:          sp,
 		Cfg:                 cfg.Language,
 		Language:            cfg.Language,
+		Timeout:             time.Duration(timeoutms) * time.Millisecond,
 	}
 
 	if cfg.Cfg.GetBool("templateMetrics") {

helpers/content.go

@@ -400,6 +400,9 @@ func (c ContentSpec) mmarkRender(ctx *RenderingContext) []byte {
 
 // ExtractTOC extracts Table of Contents from content.
 func ExtractTOC(content []byte) (newcontent []byte, toc []byte) {
+	if !bytes.Contains(content, []byte("<nav>")) {
+		return content, nil
+	}
 	origContent := make([]byte, len(content))
 	copy(origContent, content)
 	first := []byte(`<nav>

hugolib/config.go

@@ -435,6 +435,7 @@ func loadDefaultSettingsFor(v *viper.Viper) error {
 	v.SetDefault("disableAliases", false)
 	v.SetDefault("debug", false)
 	v.SetDefault("disableFastRender", false)
+	v.SetDefault("timeout", 10000) // 10 seconds
 
 	// Remove in Hugo 0.39
 

hugolib/embedded_shortcodes_test.go

@@ -69,7 +69,7 @@ func doTestShortcodeCrossrefs(t *testing.T, relative bool) {
 
 	require.Len(t, s.RegularPages, 1)
 
-	output := string(s.RegularPages[0].content)
+	output := string(s.RegularPages[0].content())
 
 	if !strings.Contains(output, expected) {
 		t.Errorf("Got\n%q\nExpected\n%q", output, expected)

hugolib/hugo_sites.go

@@ -560,45 +560,30 @@ func (h *HugoSites) setupTranslations() {
 }
 
 func (s *Site) preparePagesForRender(cfg *BuildCfg) {
-
-	pageChan := make(chan *Page)
-	wg := &sync.WaitGroup{}
-
-	numWorkers := getGoMaxProcs() * 4
-
-	for i := 0; i < numWorkers; i++ {
-		wg.Add(1)
-		go func(pages <-chan *Page, wg *sync.WaitGroup) {
-			defer wg.Done()
-			for p := range pages {
-				if err := p.prepareForRender(cfg); err != nil {
-					s.Log.ERROR.Printf("Failed to prepare page %q for render: %s", p.BaseFileName(), err)
-
-				}
-			}
-		}(pageChan, wg)
-	}
-
 	for _, p := range s.Pages {
-		pageChan <- p
+		p.setContentInit(cfg)
+		// The skip render flag is used in many tests. To make sure that they
+		// have access to the content, we need to manually initialize it here.
+		if cfg.SkipRender {
+			p.initContent()
+		}
 	}
 
 	for _, p := range s.headlessPages {
-		pageChan <- p
+		p.setContentInit(cfg)
+		if cfg.SkipRender {
+			p.initContent()
+		}
 	}
 
-	close(pageChan)
-
-	wg.Wait()
-
 }
 
 // Pages returns all pages for all sites.
 func (h *HugoSites) Pages() Pages {
 	return h.Sites[0].AllPages
 }
 
-func handleShortcodes(p *Page, rawContentCopy []byte) ([]byte, error) {
+func handleShortcodes(p *PageWithoutContent, rawContentCopy []byte) ([]byte, error) {
 	if p.shortcodeState != nil && len(p.shortcodeState.contentShortcodes) > 0 {
 		p.s.Log.DEBUG.Printf("Replace %d shortcodes in %q", len(p.shortcodeState.contentShortcodes), p.BaseFileName())
 		err := p.shortcodeState.executeShortcodesForDelta(p)

hugolib/hugo_sites_build.go

@@ -224,6 +224,7 @@ func (h *HugoSites) render(config *BuildCfg) error {
 		s.initRenderFormats()
 		for i, rf := range s.renderFormats {
 			s.rc = &siteRenderingContext{Format: rf}
+
 			s.preparePagesForRender(config)
 
 			if !config.SkipRender {

hugolib/hugo_sites_build_test.go

@@ -378,9 +378,9 @@ func doTestMultiSitesBuild(t *testing.T, configTemplate, configSuffix string) {
 	b.AssertFileContent("public/en/tags/tag1/index.html", "Tag1|Hello|http://example.com/blog/en/tags/tag1/")
 
 	// Check Blackfriday config
-	require.True(t, strings.Contains(string(doc1fr.content), "«"), string(doc1fr.content))
-	require.False(t, strings.Contains(string(doc1en.content), "«"), string(doc1en.content))
-	require.True(t, strings.Contains(string(doc1en.content), "“"), string(doc1en.content))
+	require.True(t, strings.Contains(string(doc1fr.content()), "«"), string(doc1fr.content()))
+	require.False(t, strings.Contains(string(doc1en.content()), "«"), string(doc1en.content()))
+	require.True(t, strings.Contains(string(doc1en.content()), "“"), string(doc1en.content()))
 
 	// Check that the drafts etc. are not built/processed/rendered.
 	assertShouldNotBuild(t, b.H)
@@ -630,9 +630,9 @@ func assertShouldNotBuild(t *testing.T, sites *HugoSites) {
 	for _, p := range s.rawAllPages {
 		// No HTML when not processed
 		require.Equal(t, p.shouldBuild(), bytes.Contains(p.workContent, []byte("</")), p.BaseFileName()+": "+string(p.workContent))
-		require.Equal(t, p.shouldBuild(), p.content != "", p.BaseFileName())
+		require.Equal(t, p.shouldBuild(), p.content() != "", p.BaseFileName())
 
-		require.Equal(t, p.shouldBuild(), p.content != "", p.BaseFileName())
+		require.Equal(t, p.shouldBuild(), p.content() != "", p.BaseFileName())
 
 	}
 }
@@ -753,6 +753,29 @@ var tocShortcode = `
 {{ .Page.TableOfContents }}
 `
 
+func TestSelfReferencedContentInShortcode(t *testing.T) {
+	t.Parallel()
+
+	b := newMultiSiteTestDefaultBuilder(t)
+
+	var (
+		shortcode = `{{- .Page.Content -}}{{- .Page.Summary -}}{{- .Page.Plain -}}{{- .Page.PlainWords -}}{{- .Page.WordCount -}}{{- .Page.ReadingTime -}}`
+
+		page = `---
+title: sctest
+---
+Empty:{{< mycontent >}}:
+`
+	)
+
+	b.WithTemplatesAdded("layouts/shortcodes/mycontent.html", shortcode)
+	b.WithContent("post/simple.en.md", page)
+
+	b.CreateSites().Build(BuildCfg{})
+
+	b.AssertFileContent("public/en/post/simple/index.html", "Empty:[]00:")
+}
+
 var tocPageSimple = `---
 title: tocTest
 publishdate: "2000-01-01"