Skip to content
Permalink
Browse files

Rework template handling for function and map lookups

This is a big commit, but it deletes lots of code and simplifies a lot.

* Resolving the template funcs at execution time means we don't have to create template clones per site
* Having a custom map resolver means that we can remove the AST lower case transformation for the special lower case Params map

Not only is the above easier to reason about, it's also faster, especially if you have more than one language, as in the benchmark below:

```
name                          old time/op    new time/op    delta
SiteNew/Deep_content_tree-16    53.7ms ± 0%    48.1ms ± 2%  -10.38%  (p=0.029 n=4+4)

name                          old alloc/op   new alloc/op   delta
SiteNew/Deep_content_tree-16    41.0MB ± 0%    36.8MB ± 0%  -10.26%  (p=0.029 n=4+4)

name                          old allocs/op  new allocs/op  delta
SiteNew/Deep_content_tree-16      481k ± 0%      410k ± 0%  -14.66%  (p=0.029 n=4+4)
```

This should be even better if you also have lots of templates.

Closes #6594
  • Loading branch information
bep committed Dec 10, 2019
1 parent 167c015 commit a03c631c420a03f9d90699abdf9be7e4fca0ff61
Showing with 1,124 additions and 1,828 deletions.
  1. +13 −10 commands/server.go
  2. +2 −2 create/content_template_handler.go
  3. +30 −24 deps/deps.go
  4. +12 −31 hugolib/alias.go
  5. +0 −74 hugolib/case_insensitive_test.go
  6. +2 −13 hugolib/embedded_shortcodes_test.go
  7. +2 −2 hugolib/hugo_modules_test.go
  8. +2 −2 hugolib/hugo_sites.go
  9. +21 −0 hugolib/hugo_smoke_test.go
  10. +1 −1 hugolib/page.go
  11. +2 −2 hugolib/page__per_output.go
  12. +1 −1 hugolib/page_test.go
  13. +3 −3 hugolib/shortcode.go
  14. +18 −18 hugolib/shortcode_test.go
  15. +5 −5 hugolib/site.go
  16. +1 −1 hugolib/sitemap_test.go
  17. +0 −97 hugolib/template_engines_test.go
  18. +2 −2 hugolib/testhelpers_test.go
  19. +19 −3 langs/language.go
  20. +15 −12 resources/resource_transformers/templates/execute_as_template.go
  21. +2 −0 scripts/fork_go_templates/main.go
  22. +2 −11 tpl/collections/apply.go
  23. +12 −3 tpl/collections/apply_test.go
  24. +2 −2 tpl/internal/go_templates/texttemplate/exec.go
  25. +0 −38 tpl/internal/go_templates/texttemplate/hugo_exec.go
  26. +141 −17 tpl/internal/go_templates/texttemplate/hugo_template.go
  27. +71 −0 tpl/internal/go_templates/texttemplate/hugo_template_test.go
  28. +1 −1 tpl/partials/partials.go
  29. +1 −1 tpl/resources/resources.go
  30. +19 −199 tpl/template.go
  31. +1 −0 tpl/tplimpl/embedded/templates.autogen.go
  32. +1 −0 tpl/tplimpl/embedded/templates/alias.html
  33. +0 −5 tpl/tplimpl/shortcodes.go
  34. +642 −628 tpl/tplimpl/template.go
  35. +0 −19 tpl/tplimpl/templateFuncster.go
  36. +2 −2 tpl/tplimpl/templateProvider.go
  37. +1 −202 tpl/tplimpl/template_ast_transformers.go
  38. +9 −387 tpl/tplimpl/template_ast_transformers_test.go
  39. +61 −5 tpl/tplimpl/template_funcs.go
  40. +4 −4 tpl/tplimpl/template_funcs_test.go
  41. +1 −1 tpl/tplimpl/template_info_test.go
@@ -16,6 +16,7 @@ package commands
import (
"bytes"
"fmt"
"io"
"net"
"net/http"
"net/url"
@@ -33,7 +34,6 @@ import (
"github.com/pkg/errors"

"github.com/gohugoio/hugo/livereload"
"github.com/gohugoio/hugo/tpl"

"github.com/gohugoio/hugo/config"
"github.com/gohugoio/hugo/helpers"
@@ -287,7 +287,7 @@ func getRootWatchDirsStr(baseDir string, watchDirs []string) string {
type fileServer struct {
baseURLs []string
roots []string
errorTemplate tpl.Template
errorTemplate func(err interface{}) (io.Reader, error)
c *commandeer
s *serverCmd
}
@@ -335,16 +335,15 @@ func (f *fileServer) createEndpoint(i int) (*http.ServeMux, string, string, erro
err := f.c.getErrorWithContext()
if err != nil {
w.WriteHeader(500)
var b bytes.Buffer
err := f.errorTemplate.Execute(&b, err)
r, err := f.errorTemplate(err)
if err != nil {
f.c.logger.ERROR.Println(err)
}
port = 1313
if !f.c.paused {
port = f.c.Cfg.GetInt("liveReloadPort")
}
fmt.Fprint(w, injectLiveReloadScript(&b, port))
fmt.Fprint(w, injectLiveReloadScript(r, port))

return
}
@@ -422,11 +421,15 @@ func (c *commandeer) serve(s *serverCmd) error {
}

srv := &fileServer{
baseURLs: baseURLs,
roots: roots,
c: c,
s: s,
errorTemplate: templ,
baseURLs: baseURLs,
roots: roots,
c: c,
s: s,
errorTemplate: func(ctx interface{}) (io.Reader, error) {
b := &bytes.Buffer{}
err := c.hugo().Tmpl.Execute(templ, b, ctx)
return b, err
},
}

doLiveReload := !c.Cfg.GetBool("disableLiveReload")
@@ -129,7 +129,7 @@ func executeArcheTypeAsTemplate(s *hugolib.Site, name, kind, targetPath, archety
archetypeTemplate = []byte(archetypeShortcodeReplacementsPre.Replace(string(archetypeTemplate)))

// Reuse the Hugo template setup to get the template funcs properly set up.
templateHandler := s.Deps.Tmpl.(tpl.TemplateHandler)
templateHandler := s.Deps.Tmpl.(tpl.TemplateManager)
templateName := "_text/" + helpers.Filename(archetypeFilename)
if err := templateHandler.AddTemplate(templateName, string(archetypeTemplate)); err != nil {
return nil, errors.Wrapf(err, "Failed to parse archetype file %q:", archetypeFilename)
@@ -138,7 +138,7 @@ func executeArcheTypeAsTemplate(s *hugolib.Site, name, kind, targetPath, archety
templ, _ := templateHandler.Lookup(templateName)

var buff bytes.Buffer
if err := templ.Execute(&buff, data); err != nil {
if err := templateHandler.Execute(templ, &buff, data); err != nil {
return nil, errors.Wrapf(err, "Failed to process archetype file %q:", archetypeFilename)
}

@@ -37,8 +37,8 @@ type Deps struct {
// Used to log warnings that may repeat itself many times.
DistinctWarningLog *helpers.DistinctLogger

// The templates to use. This will usually implement the full tpl.TemplateHandler.
Tmpl tpl.TemplateFinder `json:"-"`
// The templates to use. This will usually implement the full tpl.TemplateManager.
Tmpl tpl.TemplateHandler `json:"-"`

// We use this to parse and execute ad-hoc text templates.
TextTmpl tpl.TemplateParseFinder `json:"-"`
@@ -77,7 +77,10 @@ type Deps struct {
OutputFormatsConfig output.Formats

templateProvider ResourceProvider
WithTemplate func(templ tpl.TemplateHandler) error `json:"-"`
WithTemplate func(templ tpl.TemplateManager) error `json:"-"`

// Used in tests
OverloadedTemplateFuncs map[string]interface{}

translationProvider ResourceProvider

@@ -151,8 +154,8 @@ type ResourceProvider interface {
}

// TemplateHandler returns the used tpl.TemplateFinder as tpl.TemplateHandler.
func (d *Deps) TemplateHandler() tpl.TemplateHandler {
return d.Tmpl.(tpl.TemplateHandler)
func (d *Deps) TemplateHandler() tpl.TemplateManager {
return d.Tmpl.(tpl.TemplateManager)
}

// LoadResources loads translations and templates.
@@ -239,24 +242,25 @@ func New(cfg DepsCfg) (*Deps, error) {
distinctWarnLogger := helpers.NewDistinctLogger(logger.WARN)

d := &Deps{
Fs: fs,
Log: logger,
DistinctErrorLog: distinctErrorLogger,
DistinctWarningLog: distinctWarnLogger,
templateProvider: cfg.TemplateProvider,
translationProvider: cfg.TranslationProvider,
WithTemplate: cfg.WithTemplate,
PathSpec: ps,
ContentSpec: contentSpec,
SourceSpec: sp,
ResourceSpec: resourceSpec,
Cfg: cfg.Language,
Language: cfg.Language,
Site: cfg.Site,
FileCaches: fileCaches,
BuildStartListeners: &Listeners{},
Timeout: time.Duration(timeoutms) * time.Millisecond,
globalErrHandler: &globalErrHandler{},
Fs: fs,
Log: logger,
DistinctErrorLog: distinctErrorLogger,
DistinctWarningLog: distinctWarnLogger,
templateProvider: cfg.TemplateProvider,
translationProvider: cfg.TranslationProvider,
WithTemplate: cfg.WithTemplate,
OverloadedTemplateFuncs: cfg.OverloadedTemplateFuncs,
PathSpec: ps,
ContentSpec: contentSpec,
SourceSpec: sp,
ResourceSpec: resourceSpec,
Cfg: cfg.Language,
Language: cfg.Language,
Site: cfg.Site,
FileCaches: fileCaches,
BuildStartListeners: &Listeners{},
Timeout: time.Duration(timeoutms) * time.Millisecond,
globalErrHandler: &globalErrHandler{},
}

if cfg.Cfg.GetBool("templateMetrics") {
@@ -344,7 +348,9 @@ type DepsCfg struct {

// Template handling.
TemplateProvider ResourceProvider
WithTemplate func(templ tpl.TemplateHandler) error
WithTemplate func(templ tpl.TemplateManager) error
// Used in tests
OverloadedTemplateFuncs map[string]interface{}

// i18n handling.
TranslationProvider ResourceProvider
@@ -15,6 +15,7 @@ package hugolib

import (
"bytes"
"errors"
"fmt"
"html/template"
"io"
@@ -31,27 +32,15 @@ import (
"github.com/gohugoio/hugo/tpl"
)

const (
alias = "<!DOCTYPE html><html><head><title>{{ .Permalink }}</title><link rel=\"canonical\" href=\"{{ .Permalink }}\"/><meta name=\"robots\" content=\"noindex\"><meta charset=\"utf-8\" /><meta http-equiv=\"refresh\" content=\"0; url={{ .Permalink }}\" /></head></html>"
aliasXHtml = "<!DOCTYPE html><html xmlns=\"http://www.w3.org/1999/xhtml\"><head><title>{{ .Permalink }}</title><link rel=\"canonical\" href=\"{{ .Permalink }}\"/><meta name=\"robots\" content=\"noindex\"><meta http-equiv=\"content-type\" content=\"text/html; charset=utf-8\" /><meta http-equiv=\"refresh\" content=\"0; url={{ .Permalink }}\" /></head></html>"
)

var defaultAliasTemplates *template.Template

func init() {
//TODO(bep) consolidate
defaultAliasTemplates = template.New("")
template.Must(defaultAliasTemplates.New("alias").Parse(alias))
template.Must(defaultAliasTemplates.New("alias-xhtml").Parse(aliasXHtml))
}

type aliasHandler struct {
t tpl.TemplateFinder
t tpl.TemplateHandler
log *loggers.Logger
allowRoot bool
}

func newAliasHandler(t tpl.TemplateFinder, l *loggers.Logger, allowRoot bool) aliasHandler {
func newAliasHandler(t tpl.TemplateHandler, l *loggers.Logger, allowRoot bool) aliasHandler {
return aliasHandler{t, l, allowRoot}
}

@@ -60,33 +49,27 @@ type aliasPage struct {
page.Page
}

func (a aliasHandler) renderAlias(isXHTML bool, permalink string, p page.Page) (io.Reader, error) {
t := "alias"
if isXHTML {
t = "alias-xhtml"
}
func (a aliasHandler) renderAlias(permalink string, p page.Page) (io.Reader, error) {

var templ tpl.Template
var found bool

if a.t != nil {
templ, found = a.t.Lookup("alias.html")
}

templ, found = a.t.Lookup("alias.html")
if !found {
def := defaultAliasTemplates.Lookup(t)
if def != nil {
templ = &tpl.TemplateAdapter{Template: def}
// TODO(bep) consolidate
templ, found = a.t.Lookup("_internal/alias.html")
if !found {
return nil, errors.New("no alias template found")
}

}

data := aliasPage{
permalink,
p,
}

buffer := new(bytes.Buffer)
err := templ.Execute(buffer, data)
err := a.t.Execute(templ, buffer, data)
if err != nil {
return nil, err
}
@@ -100,16 +83,14 @@ func (s *Site) writeDestAlias(path, permalink string, outputFormat output.Format
func (s *Site) publishDestAlias(allowRoot bool, path, permalink string, outputFormat output.Format, p page.Page) (err error) {
handler := newAliasHandler(s.Tmpl, s.Log, allowRoot)

isXHTML := strings.HasSuffix(path, ".xhtml")

s.Log.DEBUG.Println("creating alias:", path, "redirecting to", permalink)

targetPath, err := handler.targetPathAlias(path)
if err != nil {
return err
}

aliasContent, err := handler.renderAlias(isXHTML, permalink, p)
aliasContent, err := handler.renderAlias(permalink, p)
if err != nil {
return err
}
@@ -14,7 +14,6 @@
package hugolib

import (
"fmt"
"path/filepath"
"testing"

@@ -232,76 +231,3 @@ Page2: {{ $page2.Params.ColoR }}
"index2|Site: yellow|",
)
}

// TODO1
func TestCaseInsensitiveConfigurationForAllTemplateEngines(t *testing.T) {
t.Parallel()

noOp := func(s string) string {
return s
}

for _, config := range []struct {
suffix string
templateFixer func(s string) string
}{
//{"amber", amberFixer},
{"html", noOp},
//{"ace", noOp},
} {
doTestCaseInsensitiveConfigurationForTemplateEngine(t, config.suffix, config.templateFixer)

}

}

func doTestCaseInsensitiveConfigurationForTemplateEngine(t *testing.T, suffix string, templateFixer func(s string) string) {
c := qt.New(t)
mm := afero.NewMemMapFs()

caseMixingTestsWriteCommonSources(t, mm)

cfg, err := LoadConfigDefault(mm)
c.Assert(err, qt.IsNil)

fs := hugofs.NewFrom(mm, cfg)

th := newTestHelper(cfg, fs, t)

t.Log("Testing", suffix)

templTemplate := `
p
|
| Page Colors: {{ .Params.CoLOR }}|{{ .Params.Colors.Blue }}
| Site Colors: {{ .Site.Params.COlOR }}|{{ .Site.Params.COLORS.YELLOW }}
| {{ .Content }}
`

templ := templateFixer(templTemplate)

t.Log(templ)

writeSource(t, fs, filepath.Join("layouts", "_default", fmt.Sprintf("single.%s", suffix)), templ)

sites, err := NewHugoSites(deps.DepsCfg{Fs: fs, Cfg: cfg})

if err != nil {
t.Fatalf("Failed to create sites: %s", err)
}

err = sites.Build(BuildCfg{})

if err != nil {
t.Fatalf("Failed to build sites: %s", err)
}

th.assertFileContent(filepath.Join("public", "nn", "sect1", "page1", "index.html"),
"Page Colors: red|heavenly",
"Site Colors: green|yellow",
"Shortcode Page: red|heavenly",
"Shortcode Site: green|yellow",
)

}

0 comments on commit a03c631

Please sign in to comment.
You can’t perform that action at this time.