Skip to content

Commit

Permalink
Simplify page tree logic
Browse files Browse the repository at this point in the history 
This is preparation for gohugoio#6041.

For historic reasons, the code for bulding the section tree and the taxonomies were very much separate.

This works, but makes it hard to extend, maintain, and possibly not so fast as it could be.

This simplification also introduces 3 slightly breaking changes, which I suspect most people will be pleased about. See referenced issues:

This commit also switches the radix tree dependency to a mutable implementation: github.com/armon/go-radix.

Fixes gohugoio#6154
Fixes gohugoio#6153
Fixes gohugoio#6152
  • Loading branch information
@bep
bep committed Aug 8, 2019
1 parent df37485 commit 928260a
Show file tree
Hide file tree
Showing 33 changed files with 849 additions and 833 deletions.
5 changes: 3 additions & 2 deletions common/herrors/errors.go
View file
Expand Up @@ -50,9 +50,10 @@ func FprintStackTrace(w io.Writer, err error) {
// Recover is a helper function that can be used to capture panics.
// Put this at the top of a method/function that crashes in a template:
// defer herrors.Recover()
func Recover() {
func Recover(args ...interface{}) {
if r := recover(); r != nil {
fmt.Println("stacktrace from panic: \n" + string(debug.Stack()))
args = append(args, "stacktrace from panic: \n"+string(debug.Stack()), "\n")
fmt.Println(args...)
}

}
Expand Down
28 changes: 28 additions & 0 deletions common/maps/maps_get.go
View file
@@ -0,0 +1,28 @@
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package maps

import (
"github.com/spf13/cast"
)

// GetString tries to get a value with key from map m and convert it to a string.
// It will return an empty string if not found or if it cannot be convertd to a string.
func GetString(m map[string]interface{}, key string) string {
v, found := m[key]
if !found {
return ""
}
return cast.ToString(v)
}
8 changes: 3 additions & 5 deletions go.mod
View file
Expand Up @@ -8,6 +8,7 @@ require (
github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38
github.com/alecthomas/chroma v0.6.4
github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1 // indirect
github.com/armon/go-radix v1.0.0
github.com/aws/aws-sdk-go v1.19.40
github.com/bep/debounce v1.2.0
github.com/bep/gitmap v1.1.0
Expand All @@ -17,29 +18,25 @@ require (
github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385
github.com/fortytw2/leaktest v1.3.0
github.com/fsnotify/fsnotify v1.4.7
github.com/go-errors/errors v1.0.1
github.com/gobwas/glob v0.2.3
github.com/gohugoio/testmodBuilder/mods v0.0.0-20190520184928-c56af20f2e95
github.com/google/go-cmp v0.3.0
github.com/gorilla/websocket v1.4.0
github.com/hashicorp/go-immutable-radix v1.0.0
github.com/hashicorp/go-uuid v1.0.1 // indirect
github.com/jdkato/prose v1.1.0
github.com/kyokomi/emoji v1.5.1
github.com/magefile/mage v1.4.0
github.com/magiconair/properties v1.8.1 // indirect
github.com/markbates/inflect v1.0.0
github.com/mattn/go-isatty v0.0.8
github.com/mattn/go-runewidth v0.0.4 // indirect
github.com/miekg/mmark v1.3.6
github.com/mitchellh/hashstructure v1.0.0
github.com/mitchellh/mapstructure v1.1.2
github.com/muesli/smartcrop v0.0.0-20180228075044-f6ebaa786a12
github.com/ncw/rclone v1.48.0
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect
github.com/nicksnyder/go-i18n v1.10.0
github.com/niklasfasching/go-org v0.1.2
github.com/olekukonko/tablewriter v0.0.0-20180506121414-d4647c9c7a84
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
github.com/pelletier/go-toml v1.4.0 // indirect
github.com/pkg/errors v0.8.1
github.com/rogpeppe/go-internal v1.3.0
Expand All @@ -58,6 +55,7 @@ require (
go.opencensus.io v0.22.0 // indirect
gocloud.dev v0.15.0
golang.org/x/image v0.0.0-20190523035834-f03afa92d3ff
golang.org/x/net v0.0.0-20190606173856-1492cefac77f // indirect
golang.org/x/oauth2 v0.0.0-20190523182746-aaccbc9213b0 // indirect
golang.org/x/sync v0.0.0-20190423024810-112230192c58
golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 // indirect
Expand Down
83 changes: 2 additions & 81 deletions go.sum
View file

Large diffs are not rendered by default.

18 changes: 9 additions & 9 deletions hugofs/rootmapping_fs.go
View file
Expand Up @@ -23,7 +23,7 @@ import (

"github.com/pkg/errors"

radix "github.com/hashicorp/go-immutable-radix"
radix "github.com/armon/go-radix"
"github.com/spf13/afero"
)

Expand All @@ -33,7 +33,7 @@ var filepathSeparator = string(filepath.Separator)
// of root mappings with some optional metadata about the root.
// Note that From represents a virtual root that maps to the actual filename in To.
func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) {
rootMapToReal := radix.New().Txn()
rootMapToReal := radix.New()

for _, rm := range rms {
(&rm).clean()
Expand All @@ -58,7 +58,7 @@ func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) {
// Extract "blog" from "content/blog"
rm.path = strings.TrimPrefix(strings.TrimPrefix(rm.From, fromBase), filepathSeparator)

key := []byte(rm.rootKey())
key := rm.rootKey()
var mappings []RootMapping
v, found := rootMapToReal.Get(key)
if found {
Expand All @@ -71,7 +71,7 @@ func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) {

rfs := &RootMappingFs{Fs: fs,
virtualRoots: rms,
rootMapToReal: rootMapToReal.Commit().Root()}
rootMapToReal: rootMapToReal}

return rfs, nil
}
Expand Down Expand Up @@ -119,7 +119,7 @@ func (r RootMapping) rootKey() string {
// in the order given.
type RootMappingFs struct {
afero.Fs
rootMapToReal *radix.Node
rootMapToReal *radix.Tree
virtualRoots []RootMapping
filter func(r RootMapping) bool
}
Expand Down Expand Up @@ -303,8 +303,8 @@ func (fs *RootMappingFs) isRoot(name string) bool {
}

func (fs *RootMappingFs) getRoots(name string) []RootMapping {
nameb := []byte(filepath.Clean(name))
_, v, found := fs.rootMapToReal.LongestPrefix(nameb)
name = filepath.Clean(name)
_, v, found := fs.rootMapToReal.LongestPrefix(name)
if !found {
return nil
}
Expand Down Expand Up @@ -333,10 +333,10 @@ func (fs *RootMappingFs) getRootsWithPrefix(prefix string) []RootMapping {
if fs.isRoot(prefix) {
return fs.virtualRoots
}
prefixb := []byte(filepath.Clean(prefix))
prefix = filepath.Clean(prefix)
var roots []RootMapping

fs.rootMapToReal.WalkPrefix(prefixb, func(b []byte, v interface{}) bool {
fs.rootMapToReal.WalkPrefix(prefix, func(b string, v interface{}) bool {
roots = append(roots, v.([]RootMapping)...)
return false
})
Expand Down
155 changes: 6 additions & 149 deletions hugolib/hugo_sites.go
View file
Expand Up @@ -14,15 +14,13 @@
package hugolib

import (
"fmt"
"io"
"path"
"path/filepath"
"sort"
"strings"
"sync"

radix "github.com/hashicorp/go-immutable-radix"
radix "github.com/armon/go-radix"

"github.com/gohugoio/hugo/output"
"github.com/gohugoio/hugo/parser/metadecoders"
Expand Down Expand Up @@ -623,142 +621,14 @@ func (h *HugoSites) renderCrossSitesArtifacts() error {
s.siteCfg.sitemap.Filename, h.toSiteInfos(), smLayouts...)
}

// createMissingPages creates home page, taxonomies etc. that isnt't created as an
// effect of having a content file.
func (h *HugoSites) createMissingPages() error {

for _, s := range h.Sites {
if s.isEnabled(page.KindHome) {
// home pages
homes := s.findWorkPagesByKind(page.KindHome)
if len(homes) > 1 {
panic("Too many homes")
}
var home *pageState
if len(homes) == 0 {
home = s.newPage(page.KindHome)
s.workAllPages = append(s.workAllPages, home)
} else {
home = homes[0]
}

s.home = home
}

// Will create content-less root sections.
newSections := s.assembleSections()
s.workAllPages = append(s.workAllPages, newSections...)

taxonomyTermEnabled := s.isEnabled(page.KindTaxonomyTerm)
taxonomyEnabled := s.isEnabled(page.KindTaxonomy)

// taxonomy list and terms pages
taxonomies := s.Language().GetStringMapString("taxonomies")
if len(taxonomies) > 0 {
taxonomyPages := s.findWorkPagesByKind(page.KindTaxonomy)
taxonomyTermsPages := s.findWorkPagesByKind(page.KindTaxonomyTerm)

// Make them navigable from WeightedPage etc.
for _, p := range taxonomyPages {
ni := p.getTaxonomyNodeInfo()
if ni == nil {
// This can be nil for taxonomies, e.g. an author,
// with a content file, but no actual usage.
// Create one.
sections := p.SectionsEntries()
if len(sections) < 2 {
// Invalid state
panic(fmt.Sprintf("invalid taxonomy state for %q with sections %v", p.pathOrTitle(), sections))
}
ni = p.s.taxonomyNodes.GetOrAdd(sections[0], path.Join(sections[1:]...))
}
ni.TransferValues(p)
}
for _, p := range taxonomyTermsPages {
p.getTaxonomyNodeInfo().TransferValues(p)
}

for _, plural := range taxonomies {
if taxonomyTermEnabled {
foundTaxonomyTermsPage := false
for _, p := range taxonomyTermsPages {
if p.SectionsPath() == plural {
foundTaxonomyTermsPage = true
break
}
}

if !foundTaxonomyTermsPage {
n := s.newPage(page.KindTaxonomyTerm, plural)
n.getTaxonomyNodeInfo().TransferValues(n)
s.workAllPages = append(s.workAllPages, n)
}
}

if taxonomyEnabled {
for termKey := range s.Taxonomies[plural] {

foundTaxonomyPage := false

for _, p := range taxonomyPages {
sectionsPath := p.SectionsPath()

if !strings.HasPrefix(sectionsPath, plural) {
continue
}

singularKey := strings.TrimPrefix(sectionsPath, plural)
singularKey = strings.TrimPrefix(singularKey, "/")

if singularKey == termKey {
foundTaxonomyPage = true
break
}
}

if !foundTaxonomyPage {
info := s.taxonomyNodes.Get(plural, termKey)
if info == nil {
panic("no info found")
}

n := s.newTaxonomyPage(info.term, info.plural, info.termKey)
info.TransferValues(n)
s.workAllPages = append(s.workAllPages, n)
}
}
}
}
}
}

return nil
}

func (h *HugoSites) removePageByFilename(filename string) {
for _, s := range h.Sites {
s.removePageFilename(filename)
}
}

// TODO(bep) cm
func (h *HugoSites) createPageCollections() error {
for _, s := range h.Sites {
for _, p := range s.rawAllPages {
if !s.isEnabled(p.Kind()) {
continue
}

shouldBuild := s.shouldBuild(p)
s.buildStats.update(p)
if shouldBuild {
if p.m.headless {
s.headlessPages = append(s.headlessPages, p)
} else {
s.workAllPages = append(s.workAllPages, p)
}
}
}
}

allPages := newLazyPagesFactory(func() page.Pages {
var pages page.Pages
Expand Down Expand Up @@ -950,8 +820,7 @@ type contentChangeMap struct {
mu sync.RWMutex

// Holds directories with leaf bundles.
leafBundles *radix.Tree
leafBundlesTxn *radix.Txn
leafBundles *radix.Tree

// Holds directories with branch bundles.
branchBundles map[string]bool
Expand All @@ -969,18 +838,6 @@ type contentChangeMap struct {
symContent map[string]map[string]bool
}

func (m *contentChangeMap) start() {
m.mu.Lock()
m.leafBundlesTxn = m.leafBundles.Txn()
m.mu.Unlock()
}

func (m *contentChangeMap) stop() {
m.mu.Lock()
m.leafBundles = m.leafBundlesTxn.Commit()
m.mu.Unlock()
}

func (m *contentChangeMap) add(filename string, tp bundleDirType) {
m.mu.Lock()
dir := filepath.Dir(filename) + helpers.FilePathSeparator
Expand All @@ -989,7 +846,7 @@ func (m *contentChangeMap) add(filename string, tp bundleDirType) {
case bundleBranch:
m.branchBundles[dir] = true
case bundleLeaf:
m.leafBundlesTxn.Insert([]byte(dir), true)
m.leafBundles.Insert(dir, true)
default:
panic("invalid bundle type")
}
Expand All @@ -1012,8 +869,8 @@ func (m *contentChangeMap) resolveAndRemove(filename string) (string, string, bu
return dir, dir, bundleBranch
}

if key, _, found := m.leafBundles.Root().LongestPrefix([]byte(dir)); found {
m.leafBundlesTxn.Delete(key)
if key, _, found := m.leafBundles.LongestPrefix(dir); found {
m.leafBundles.Delete(key)
dir = string(key)
return dir, dir, bundleLeaf
}
Expand Down

0 comments on commit 928260a

Please sign in to comment.