Simplify page tree logic

This is preparation for gohugoio#6041.

For historic reasons, the code for bulding the section tree and the taxonomies were very much separate.

This works, but makes it hard to extend, maintain, and possibly not so fast as it could be.

This simplification also introduces 3 slightly breaking changes, which I suspect most people will be pleased about. See referenced issues:

Fixes gohugoio#6154
Fixes gohugoio#6153
Fixes gohugoio#6152

common/herrors/errors.go

@@ -50,9 +50,10 @@ func FprintStackTrace(w io.Writer, err error) {
 // Recover is a helper function that can be used to capture panics.
 // Put this at the top of a method/function that crashes in a template:
 //     defer herrors.Recover()
-func Recover() {
+func Recover(args ...interface{}) {
 	if r := recover(); r != nil {
-		fmt.Println("stacktrace from panic: \n" + string(debug.Stack()))
+		args = append(args, "stacktrace from panic: \n"+string(debug.Stack()), "\n")
+		fmt.Println(args...)
 	}
 
 }

common/maps/maps_get.go

@@ -0,0 +1,28 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package maps
+
+import (
+	"github.com/spf13/cast"
+)
+
+// GetString tries to get a value with key from map m and convert it to a string.
+// It will return an empty string if not found or if it cannot be convertd to a string.
+func GetString(m map[string]interface{}, key string) string {
+	v, found := m[key]
+	if !found {
+		return ""
+	}
+	return cast.ToString(v)
+}

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38
 	github.com/alecthomas/chroma v0.6.4
 	github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1 // indirect
+	github.com/armon/go-radix v1.0.0
 	github.com/aws/aws-sdk-go v1.19.40
 	github.com/bep/debounce v1.2.0
 	github.com/bep/gitmap v1.1.0

go.sum

@@ -58,6 +58,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
 github.com/anacrolix/dms v0.0.0-20180117034613-8af4925bffb5/go.mod h1:DGqLjaZ3ziKKNRt+U5Q9PLWJ52Q/4rxfaaH/b3QYKaE=
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI=
+github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
 github.com/aws/aws-sdk-go v1.18.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.19.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=

hugolib/hugo_sites.go

@@ -14,9 +14,7 @@
 package hugolib
 
 import (
-	"fmt"
 	"io"
-	"path"
 	"path/filepath"
 	"sort"
 	"strings"
@@ -623,142 +621,14 @@ func (h *HugoSites) renderCrossSitesArtifacts() error {
 		s.siteCfg.sitemap.Filename, h.toSiteInfos(), smLayouts...)
 }
 
-// createMissingPages creates home page, taxonomies etc. that isnt't created as an
-// effect of having a content file.
-func (h *HugoSites) createMissingPages() error {
-
-	for _, s := range h.Sites {
-		if s.isEnabled(page.KindHome) {
-			// home pages
-			homes := s.findWorkPagesByKind(page.KindHome)
-			if len(homes) > 1 {
-				panic("Too many homes")
-			}
-			var home *pageState
-			if len(homes) == 0 {
-				home = s.newPage(page.KindHome)
-				s.workAllPages = append(s.workAllPages, home)
-			} else {
-				home = homes[0]
-			}
-
-			s.home = home
-		}
-
-		// Will create content-less root sections.
-		newSections := s.assembleSections()
-		s.workAllPages = append(s.workAllPages, newSections...)
-
-		taxonomyTermEnabled := s.isEnabled(page.KindTaxonomyTerm)
-		taxonomyEnabled := s.isEnabled(page.KindTaxonomy)
-
-		// taxonomy list and terms pages
-		taxonomies := s.Language().GetStringMapString("taxonomies")
-		if len(taxonomies) > 0 {
-			taxonomyPages := s.findWorkPagesByKind(page.KindTaxonomy)
-			taxonomyTermsPages := s.findWorkPagesByKind(page.KindTaxonomyTerm)
-
-			// Make them navigable from WeightedPage etc.
-			for _, p := range taxonomyPages {
-				ni := p.getTaxonomyNodeInfo()
-				if ni == nil {
-					// This can be nil for taxonomies, e.g. an author,
-					// with a content file, but no actual usage.
-					// Create one.
-					sections := p.SectionsEntries()
-					if len(sections) < 2 {
-						// Invalid state
-						panic(fmt.Sprintf("invalid taxonomy state for %q with sections %v", p.pathOrTitle(), sections))
-					}
-					ni = p.s.taxonomyNodes.GetOrAdd(sections[0], path.Join(sections[1:]...))
-				}
-				ni.TransferValues(p)
-			}
-			for _, p := range taxonomyTermsPages {
-				p.getTaxonomyNodeInfo().TransferValues(p)
-			}
-
-			for _, plural := range taxonomies {
-				if taxonomyTermEnabled {
-					foundTaxonomyTermsPage := false
-					for _, p := range taxonomyTermsPages {
-						if p.SectionsPath() == plural {
-							foundTaxonomyTermsPage = true
-							break
-						}
-					}
-
-					if !foundTaxonomyTermsPage {
-						n := s.newPage(page.KindTaxonomyTerm, plural)
-						n.getTaxonomyNodeInfo().TransferValues(n)
-						s.workAllPages = append(s.workAllPages, n)
-					}
-				}
-
-				if taxonomyEnabled {
-					for termKey := range s.Taxonomies[plural] {
-
-						foundTaxonomyPage := false
-
-						for _, p := range taxonomyPages {
-							sectionsPath := p.SectionsPath()
-
-							if !strings.HasPrefix(sectionsPath, plural) {
-								continue
-							}
-
-							singularKey := strings.TrimPrefix(sectionsPath, plural)
-							singularKey = strings.TrimPrefix(singularKey, "/")
-
-							if singularKey == termKey {
-								foundTaxonomyPage = true
-								break
-							}
-						}
-
-						if !foundTaxonomyPage {
-							info := s.taxonomyNodes.Get(plural, termKey)
-							if info == nil {
-								panic("no info found")
-							}
-
-							n := s.newTaxonomyPage(info.term, info.plural, info.termKey)
-							info.TransferValues(n)
-							s.workAllPages = append(s.workAllPages, n)
-						}
-					}
-				}
-			}
-		}
-	}
-
-	return nil
-}
-
 func (h *HugoSites) removePageByFilename(filename string) {
 	for _, s := range h.Sites {
 		s.removePageFilename(filename)
 	}
 }
 
+// TODO(bep) cm
 func (h *HugoSites) createPageCollections() error {
-	for _, s := range h.Sites {
-		for _, p := range s.rawAllPages {
-			if !s.isEnabled(p.Kind()) {
-				continue
-			}
-
-			shouldBuild := s.shouldBuild(p)
-			s.buildStats.update(p)
-			if shouldBuild {
-				if p.m.headless {
-					s.headlessPages = append(s.headlessPages, p)
-				} else {
-					s.workAllPages = append(s.workAllPages, p)
-				}
-			}
-		}
-	}
 
 	allPages := newLazyPagesFactory(func() page.Pages {
 		var pages page.Pages

hugolib/hugo_sites_build.go

@@ -18,7 +18,6 @@ import (
 	"context"
 	"fmt"
 	"runtime/trace"
-	"sort"
 
 	"github.com/gohugoio/hugo/output"
 
@@ -235,26 +234,41 @@ func (h *HugoSites) assemble(config *BuildCfg) error {
 		}
 	}
 
-	if err := h.createPageCollections(); err != nil {
-		return err
-	}
-
 	if config.whatChanged.source {
-		for _, s := range h.Sites {
-			if err := s.assembleTaxonomies(); err != nil {
-				return err
-			}
-		}
+		//for _, s := range h.Sites {
+		// TODO(bep) cm
+		//if err := s.assembleTaxonomies(); err != nil {
+		//	return err
+		//}
+		//}
 	}
 
-	// Create pagexs for the section pages etc. without content file.
-	if err := h.createMissingPages(); err != nil {
+	// Create pages for the section pages etc. without content file.
+	// TODO(bep) cm
+	/*if err := h.createMissingPages(); err != nil {
 		return err
-	}
+	}*/
 
 	for _, s := range h.Sites {
-		s.setupSitePages()
-		sort.Stable(s.workAllPages)
+		if err := s.assemblePagesMap(s); err != nil {
+			return err
+		}
+
+		if err := s.pagesMap.assembleTaxonomies(s); err != nil {
+			return err
+		}
+
+		if err := s.createWorkAllPages(); err != nil {
+			return err
+		}
+
+		// TODO(bep) cm
+		//s.setupSitePages()
+		//sort.Stable(s.workAllPages)
+	}
+
+	if err := h.createPageCollections(); err != nil {
+		return err
 	}
 
 	return nil

hugolib/hugo_sites_build_test.go

@@ -365,7 +365,6 @@ func doTestMultiSitesBuild(t *testing.T, configTemplate, configSuffix string) {
 	require.NotNil(t, enTags["tag1"])
 	require.NotNil(t, frTags["FRtag1"])
 	b.AssertFileContent("public/fr/plaques/FRtag1/index.html", "FRtag1|Bonjour|http://example.com/blog/fr/plaques/FRtag1/")
-	b.AssertFileContent("public/en/tags/tag1/index.html", "tag1|Hello|http://example.com/blog/en/tags/tag1/")
 
 	// Check Blackfriday config
 	require.True(t, strings.Contains(content(doc1fr), "«"), content(doc1fr))
@@ -470,13 +469,6 @@ func TestMultiSitesRebuild(t *testing.T) {
 			func(t *testing.T) {
 				assert.Len(enSite.RegularPages(), 4, "1 en removed")
 
-				// Check build stats
-				require.Equal(t, 1, enSite.buildStats.draftCount, "Draft")
-				require.Equal(t, 1, enSite.buildStats.futureCount, "Future")
-				require.Equal(t, 1, enSite.buildStats.expiredCount, "Expired")
-				require.Equal(t, 0, frSite.buildStats.draftCount, "Draft")
-				require.Equal(t, 1, frSite.buildStats.futureCount, "Future")
-				require.Equal(t, 1, frSite.buildStats.expiredCount, "Expired")
 			},
 		},
 		{
@@ -775,13 +767,13 @@ END
 }
 
 func checkContent(s *sitesBuilder, filename string, matches ...string) {
+	s.T.Helper()
 	content := readDestination(s.T, s.Fs, filename)
 	for _, match := range matches {
 		if !strings.Contains(content, match) {
 			s.Fatalf("No match for %q in content for %s\n%q", match, filename, content)
 		}
 	}
-
 }
 
 func TestTranslationsFromContentToNonContent(t *testing.T) {

hugolib/hugo_sites_rebuild_test.go

@@ -54,7 +54,7 @@ Content.
 {{ range (.Paginate .Site.RegularPages).Pages }}
 * Page Paginate: {{ .Title }}|Summary: {{ .Summary }}|Content: {{ .Content }}
 {{ end }}
-{{ range .Pages }}
+{{ range .Site.RegularPages }}
 * Page Pages: {{ .Title }}|Summary: {{ .Summary }}|Content: {{ .Content }}
 {{ end }}
 `)

hugolib/hugo_smoke_test.go

@@ -143,8 +143,8 @@ Some **Markdown** in JSON shortcode.
 	const (
 		commonPageTemplate            = `|{{ .Kind }}|{{ .Title }}|{{ .Path }}|{{ .Summary }}|{{ .Content }}|RelPermalink: {{ .RelPermalink }}|WordCount: {{ .WordCount }}|Pages: {{ .Pages }}|Data Pages: Pages({{ len .Data.Pages }})|Resources: {{ len .Resources }}|Summary: {{ .Summary }}`
 		commonPaginatorTemplate       = `|Paginator: {{ with .Paginator }}{{ .PageNumber }}{{ else }}NIL{{ end }}`
-		commonListTemplateNoPaginator = `|{{ range $i, $e := (.Pages | first 1) }}|Render {{ $i }}: {{ .Kind }}|{{ .Render "li" }}|{{ end }}|Site params: {{ $.Site.Params.hugo }}|RelPermalink: {{ .RelPermalink }}`
-		commonListTemplate            = commonPaginatorTemplate + `|{{ range $i, $e := (.Pages | first 1) }}|Render {{ $i }}: {{ .Kind }}|{{ .Render "li" }}|{{ end }}|Site params: {{ $.Site.Params.hugo }}|RelPermalink: {{ .RelPermalink }}`
+		commonListTemplateNoPaginator = `|{{ $pages := .Pages }}{{ if .IsHome }}{{ $pages = .Site.RegularPages }}{{ end }}{{ range $i, $e := ($pages | first 1) }}|Render {{ $i }}: {{ .Kind }}|{{ .Render "li" }}|{{ end }}|Site params: {{ $.Site.Params.hugo }}|RelPermalink: {{ .RelPermalink }}`
+		commonListTemplate            = commonPaginatorTemplate + `|{{ $pages := .Pages }}{{ if .IsHome }}{{ $pages = .Site.RegularPages }}{{ end }}{{ range $i, $e := ($pages | first 1) }}|Render {{ $i }}: {{ .Kind }}|{{ .Render "li" }}|{{ end }}|Site params: {{ $.Site.Params.hugo }}|RelPermalink: {{ .RelPermalink }}`
 		commonShortcodeTemplate       = `|{{ .Name }}|{{ .Ordinal }}|{{ .Page.Summary }}|{{ .Page.Content }}|WordCount: {{ .Page.WordCount }}`
 		prevNextTemplate              = `|Prev: {{ with .Prev }}{{ .RelPermalink }}{{ end }}|Next: {{ with .Next }}{{ .RelPermalink }}{{ end }}`
 		prevNextInSectionTemplate     = `|PrevInSection: {{ with .PrevInSection }}{{ .RelPermalink }}{{ end }}|NextInSection: {{ with .NextInSection }}{{ .RelPermalink }}{{ end }}`
@@ -193,7 +193,7 @@ Some **Markdown** in JSON shortcode.
 	b.AssertFileContent("public/index.html",
 		"home|In English",
 		"Site params: Rules",
-		"Pages: Pages(18)|Data Pages: Pages(18)",
+		"Pages: Pages(6)|Data Pages: Pages(6)",
 		"Paginator: 1",
 		"First Site: In English",
 		"RelPermalink: /",