From c069a84e253eaa4dbe01ef9f23c1b50297501d3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Sat, 11 May 2024 10:52:13 +0200 Subject: [PATCH] Tests --- hugolib/content_map.go | 37 ++++------ hugolib/content_map_page.go | 2 +- hugolib/doctree/treeshifttree.go | 7 +- hugolib/hugo_sites_build.go | 51 +++++++++++-- hugolib/pagesfromdata/pagesfromgotmpl.go | 72 +++++++++++++------ .../pagesfromgotmpl_integration_test.go | 13 ++-- 6 files changed, 125 insertions(+), 57 deletions(-) diff --git a/hugolib/content_map.go b/hugolib/content_map.go index a180cd0d15b..e1c546eca6a 100644 --- a/hugolib/content_map.go +++ b/hugolib/content_map.go @@ -297,7 +297,7 @@ func (m *pageMap) AddFi(fi hugofs.FileMetaInfo, whatChanged *whatChanged) (pageC _, n, replaced := s.pageMap.treePages.InsertIntoValuesDimensionWithLock(pi.Base(), ps) if h.isRebuild() && replaced { - pt.BuildState.ChangedIdentities = append(pt.BuildState.ChangedIdentities, n.GetIdentity()) + pt.AddChange(n.GetIdentity()) } return nil @@ -313,7 +313,7 @@ func (m *pageMap) AddFi(fi hugofs.FileMetaInfo, whatChanged *whatChanged) (pageC _, n, replaced := s.pageMap.treeResources.InsertIntoValuesDimensionWithLock(rc.Path, rs) if h.isRebuild() && replaced { - pt.BuildState.ChangedIdentities = append(pt.BuildState.ChangedIdentities, n.GetIdentity()) + pt.AddChange(n.GetIdentity()) } return nil }, @@ -324,14 +324,20 @@ func (m *pageMap) AddFi(fi hugofs.FileMetaInfo, whatChanged *whatChanged) (pageC } - contentAdapter.BuildState.PrepareNextBuild() + handleBuildInfo := func(s *Site, bi pagesfromdata.BuildInfo) { + resourceCount += bi.NumResourcesAdded + pageCount += bi.NumPagesAdded + s.handleContentAdapterChanges(bi, whatChanged) + } // TODO1 do we need a mutex? - if err := contentAdapter.Execute(context.Background()); err != nil { + bi, err := contentAdapter.Execute(context.Background()) + if err != nil { return err } + handleBuildInfo(s, bi) - if !rebuild && contentAdapter.BuildState.EnableAllLanguages { + if !rebuild && bi.EnableAllLanguages { // Clone and insert the adapter for the other sites. for _, ss := range s.h.Sites { if s == ss { @@ -341,9 +347,11 @@ func (m *pageMap) AddFi(fi hugofs.FileMetaInfo, whatChanged *whatChanged) (pageC clone := contentAdapter.CloneForSite(ss) // Make sure it gets executed for the first time. - if err := clone.Execute(context.Background()); err != nil { + bi, err := clone.Execute(context.Background()) + if err != nil { return err } + handleBuildInfo(ss, bi) // Insert into the correct language tree so it get rebuilt on changes. ss.pageMap.treePagesFromTemplateOptions.Insert(pi.Base(), clone) @@ -351,23 +359,6 @@ func (m *pageMap) AddFi(fi hugofs.FileMetaInfo, whatChanged *whatChanged) (pageC } } - resourceCount += contentAdapter.BuildState.NumResourcesAdded - pageCount += contentAdapter.BuildState.NumPagesAdded - - if m.s.h.isRebuild() { - if len(contentAdapter.BuildState.ChangedIdentities) > 0 { - whatChanged.Add(contentAdapter.BuildState.ChangedIdentities...) - } - - for _, p := range contentAdapter.BuildState.DeletedPaths { - // TODO1 language, resource etc. - pp := path.Join(pi.Base(), p) - if v, ok := m.treePages.Delete(pp); ok { - whatChanged.Add(v.GetIdentity()) - } - - } - } return nil }(); err != nil { addErr = err diff --git a/hugolib/content_map_page.go b/hugolib/content_map_page.go index df32751f117..a34da035793 100644 --- a/hugolib/content_map_page.go +++ b/hugolib/content_map_page.go @@ -1151,7 +1151,7 @@ func (h *HugoSites) resolveAndClearStateForIdentities( for _, id := range changes { checkedCount++ if r := depsFinder.Contains(id, n.DependencyManager, 2); r > identity.FinderNotFound { - n.BuildState.Rebuild = true + n.MarkStale() matchCount++ break } diff --git a/hugolib/doctree/treeshifttree.go b/hugolib/doctree/treeshifttree.go index e1533605487..cd13b9f6102 100644 --- a/hugolib/doctree/treeshifttree.go +++ b/hugolib/doctree/treeshifttree.go @@ -55,10 +55,13 @@ func (t *TreeShiftTree[T]) Get(s string) T { return t.trees[t.v].Get(s) } -func (t *TreeShiftTree[T]) GetAll(s string, f func(v T)) { +func (t *TreeShiftTree[T]) DeleteAllFunc(s string, f func(s string, v T) bool) { for _, tt := range t.trees { if v := tt.Get(s); v != t.zero { - f(v) + if f(s, v) { + // Delete. + tt.tree.Delete(s) + } } } } diff --git a/hugolib/hugo_sites_build.go b/hugolib/hugo_sites_build.go index c3198c9b7a2..2694a715df1 100644 --- a/hugolib/hugo_sites_build.go +++ b/hugolib/hugo_sites_build.go @@ -253,6 +253,14 @@ func (h *HugoSites) assemble(ctx context.Context, l logg.LevelLogger, bcfg *Buil defer loggers.TimeTrackf(l, time.Now(), nil, "") if !bcfg.whatChanged.contentChanged { + changes := bcfg.whatChanged.Changes() + + if len(changes) > 0 { + // TODO1 consolidate. + if err := h.resolveAndClearStateForIdentities(ctx, l, nil, changes); err != nil { + return err + } + } return nil } @@ -709,9 +717,17 @@ func (h *HugoSites) processPartial(ctx context.Context, l logg.LevelLogger, conf changes = append(changes, ids...) } } else { - h.pageTrees.treePagesFromTemplateOptions.GetAll(pathInfo.Base(), - func(n *pagesfromdata.PagesFromTemplate) { + h.pageTrees.treePagesFromTemplateOptions.DeleteAllFunc(pathInfo.Base(), + func(s string, n *pagesfromdata.PagesFromTemplate) bool { changes = append(changes, n.DependencyManager) + + // Try to open the file to see if has been deleted. + f, err := n.GoTmplFi.Meta().Open() + if err == nil { + f.Close() + } + // TODO1 also remove all pages and resources below this path. + return err != nil }) } @@ -928,7 +944,7 @@ func (h *HugoSites) processPartial(ctx context.Context, l logg.LevelLogger, conf } if h.isRebuild() { - if err := h.processContentAdaptersOnRebuild(ctx); err != nil { + if err := h.processContentAdaptersOnRebuild(ctx, config); err != nil { return err } } @@ -955,16 +971,39 @@ func (h *HugoSites) processFull(ctx context.Context, l logg.LevelLogger, config return err } -func (h *HugoSites) processContentAdaptersOnRebuild(ctx context.Context) error { +func (s *Site) handleContentAdapterChanges(bi pagesfromdata.BuildInfo, whatChanged *whatChanged) { + if !s.h.isRebuild() { + return + } + + if len(bi.ChangedIdentities) > 0 { + whatChanged.Add(bi.ChangedIdentities...) + } + + for _, p := range bi.DeletedPaths { + pp := path.Join(bi.Path.Base(), p) + if v, ok := s.pageMap.treePages.Delete(pp); ok { + whatChanged.Add(v.GetIdentity()) + } + } +} + +func (h *HugoSites) processContentAdaptersOnRebuild(ctx context.Context, buildConfig *BuildCfg) error { g := rungroup.Run[*pagesfromdata.PagesFromTemplate](ctx, rungroup.Config[*pagesfromdata.PagesFromTemplate]{ NumWorkers: h.numWorkers, Handle: func(ctx context.Context, p *pagesfromdata.PagesFromTemplate) error { - return p.Execute(ctx) + bi, err := p.Execute(ctx) + if err != nil { + return err + } + s := p.Site.(*Site) + s.handleContentAdapterChanges(bi, buildConfig.whatChanged) + return nil }, }) h.pageTrees.treePagesFromTemplateOptions.WalkPrefixRaw(doctree.LockTypeRead, "", func(key string, p *pagesfromdata.PagesFromTemplate) (bool, error) { - if p.BuildState.Rebuild { + if p.StaleVersion() > 0 { g.Enqueue(p) } return false, nil diff --git a/hugolib/pagesfromdata/pagesfromgotmpl.go b/hugolib/pagesfromdata/pagesfromgotmpl.go index c24bc0b836a..6dc2c25a7ed 100644 --- a/hugolib/pagesfromdata/pagesfromgotmpl.go +++ b/hugolib/pagesfromdata/pagesfromgotmpl.go @@ -20,11 +20,13 @@ import ( "path/filepath" "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/common/paths" "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/identity" "github.com/gohugoio/hugo/resources/page" "github.com/gohugoio/hugo/resources/page/pagemeta" + "github.com/gohugoio/hugo/resources/resource" "github.com/gohugoio/hugo/tpl" "github.com/mitchellh/mapstructure" "github.com/spf13/cast" @@ -81,7 +83,7 @@ func (p *pagesFromDataTemplateContext) AddPage(v any) (string, error) { return "", err } - if !p.p.BuildState.checkHasChangedAndSetSourceInfo(path, m) { + if !p.p.buildState.checkHasChangedAndSetSourceInfo(path, m) { return "", nil } @@ -96,7 +98,7 @@ func (p *pagesFromDataTemplateContext) AddPage(v any) (string, error) { // TODO1 pd.Build = pagemeta.DefaultBuildConfig - p.p.BuildState.NumPagesAdded++ + p.p.buildState.NumPagesAdded++ if err := pd.Validate(true); err != nil { return "", err @@ -111,7 +113,7 @@ func (p *pagesFromDataTemplateContext) AddResource(v any) (string, error) { return "", err } - if !p.p.BuildState.checkHasChangedAndSetSourceInfo(path, m) { + if !p.p.buildState.checkHasChangedAndSetSourceInfo(path, m) { return "", nil } @@ -120,7 +122,7 @@ func (p *pagesFromDataTemplateContext) AddResource(v any) (string, error) { return "", err } - p.p.BuildState.NumResourcesAdded++ + p.p.buildState.NumResourcesAdded++ if err := rd.Validate(); err != nil { return "", err @@ -138,7 +140,7 @@ func (p *pagesFromDataTemplateContext) Store() *maps.Scratch { } func (p *pagesFromDataTemplateContext) EnableAllLanguages() string { - p.p.BuildState.EnableAllLanguages = true + p.p.buildState.EnableAllLanguages = true return "" } @@ -146,7 +148,7 @@ func NewPagesFromTemplate(opts PagesFromTemplateOptions) *PagesFromTemplate { return &PagesFromTemplate{ PagesFromTemplateOptions: opts, PagesFromTemplateDeps: opts.DepsFromSite(opts.Site), - BuildState: &BuildState{ + buildState: &BuildState{ sourceInfosCurrent: maps.NewCache[string, *sourceInfo](), }, store: maps.NewScratch(), @@ -172,15 +174,38 @@ type PagesFromTemplateDeps struct { TmplExec tpl.TemplateExecutor } +var _ resource.Staler = (*PagesFromTemplate)(nil) + type PagesFromTemplate struct { PagesFromTemplateOptions PagesFromTemplateDeps - BuildState *BuildState + buildState *BuildState store *maps.Scratch } +func (b *PagesFromTemplate) AddChange(id identity.Identity) { + b.buildState.ChangedIdentities = append(b.buildState.ChangedIdentities, id) +} + +func (b *PagesFromTemplate) MarkStale() { + b.buildState.StaleVersion++ +} + +func (b *PagesFromTemplate) StaleVersion() uint32 { + return b.buildState.StaleVersion +} + +type BuildInfo struct { + NumPagesAdded uint64 + NumResourcesAdded uint64 + EnableAllLanguages bool + ChangedIdentities []identity.Identity + DeletedPaths []string + Path *paths.Path +} + type BuildState struct { - Rebuild bool + StaleVersion uint32 EnableAllLanguages bool @@ -249,12 +274,10 @@ func (b *BuildState) resolveDeletedPaths() { b.DeletedPaths = paths } -func (b *BuildState) prepareNextExecute() { +func (b *BuildState) PrepareNextBuild() { b.sourceInfosPrevious = b.sourceInfosCurrent b.sourceInfosCurrent = maps.NewCache[string, *sourceInfo]() -} - -func (b *BuildState) PrepareNextBuild() { + b.StaleVersion = 0 b.DeletedPaths = nil b.ChangedIdentities = nil b.NumPagesAdded = 0 @@ -269,7 +292,7 @@ func (p PagesFromTemplate) CloneForSite(s page.Site) *PagesFromTemplate { // We deliberately make them share the same DepenencyManager and Store. p.PagesFromTemplateOptions.Site = s p.PagesFromTemplateDeps = p.PagesFromTemplateOptions.DepsFromSite(s) - p.BuildState = &BuildState{ + p.buildState = &BuildState{ sourceInfosCurrent: maps.NewCache[string, *sourceInfo](), } return &p @@ -284,20 +307,20 @@ func (p *PagesFromTemplate) GetDependencyManagerForScope(scope int) identity.Man return p.DependencyManager } -func (p *PagesFromTemplate) Execute(ctx context.Context) error { +func (p *PagesFromTemplate) Execute(ctx context.Context) (BuildInfo, error) { defer func() { - p.BuildState.prepareNextExecute() + p.buildState.PrepareNextBuild() }() f, err := p.GoTmplFi.Meta().Open() if err != nil { - return err + return BuildInfo{}, err } defer f.Close() tmpl, err := p.TmplFinder.Parse(filepath.ToSlash(p.GoTmplFi.Meta().Filename), helpers.ReaderToString(f)) if err != nil { - return err + return BuildInfo{}, err } data := &pagesFromDataTemplateContext{ @@ -307,13 +330,22 @@ func (p *PagesFromTemplate) Execute(ctx context.Context) error { ctx = tpl.Context.DependencyManagerScopedProvider.Set(ctx, p) if err := p.TmplExec.ExecuteWithContext(ctx, tmpl, io.Discard, data); err != nil { - return err + return BuildInfo{}, err } - p.BuildState.resolveDeletedPaths() + p.buildState.resolveDeletedPaths() // p.BuildState.printDebug() - return nil + bi := BuildInfo{ + NumPagesAdded: p.buildState.NumPagesAdded, + NumResourcesAdded: p.buildState.NumResourcesAdded, + EnableAllLanguages: p.buildState.EnableAllLanguages, + ChangedIdentities: p.buildState.ChangedIdentities, + DeletedPaths: p.buildState.DeletedPaths, + Path: p.GoTmplFi.Meta().PathInfo, + } + + return bi, nil } ////////////// diff --git a/hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go b/hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go index ad70f7fa433..3e5f3dc1273 100644 --- a/hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go +++ b/hugolib/pagesfromdata/pagesfromgotmpl_integration_test.go @@ -70,8 +70,8 @@ Pfile Content {{ $contentMarkdownDefault := dict "value" "**Hello World Default**" }} {{ $contentHTML := dict "value" "Hello World! No **markdown** here." "mediaType" "text/html" }} {{ $.AddPage (dict "kind" "page" "path" "p1" "title" $title "dates" $dates "content" $contentMarkdown "params" (dict "param1" "param1v" ) ) }} -{{ $.AddPage (dict "kind" "page" "path" "p2" "title" "p2" "dates" $dates "content" $contentHTML ) }} -{{ $.AddPage (dict "kind" "page" "path" "p3" "title" $title "dates" $dates "content" $contentMarkdownDefault ) }} +{{ $.AddPage (dict "kind" "page" "path" "p2" "title" "p2title" "dates" $dates "content" $contentHTML ) }} +{{ $.AddPage (dict "kind" "page" "path" "p3" "title" "p3title" "dates" $dates "content" $contentMarkdownDefault ) }} {{ $resourceContent := dict "type" "resource" "value" $dataResource }} {{ $.AddResource (dict "path" "p1/data1.yaml" "content" $resourceContent) }} @@ -239,7 +239,8 @@ func TestPagesFromGoTmplEditDataResource(t *testing.T) { b.AssertRenderCountPage(6) b.EditFileReplaceAll("assets/mydata.yaml", "p1: \"p1\"", "p1: \"p1edited\"").Build() b.AssertFileContent("public/docs/p1/index.html", "Single: p1edited:p1|") - b.AssertRenderCountPage(2) + b.AssertFileContent("public/docs/index.html", "p1edited") + b.AssertRenderCountPage(3) } func TestPagesFromGoTmplEditPartial(t *testing.T) { @@ -247,14 +248,15 @@ func TestPagesFromGoTmplEditPartial(t *testing.T) { b := hugolib.TestRunning(t, filesPagesFromDataTempleBasic) b.EditFileReplaceAll("layouts/partials/get-value.html", "p1", "p1edited").Build() b.AssertFileContent("public/docs/p1/index.html", "Single: p1:p1edited|") + b.AssertFileContent("public/docs/index.html", "p1edited") } func TestPagesFromGoTmplRemovePage(t *testing.T) { t.Parallel() b := hugolib.TestRunning(t, filesPagesFromDataTempleBasic) // RegularPagesRecursive: p1:p1:/docs/p1|p1:p1:/docs/p3|p2:/docs/p2|pfile:/docs/pfile|$ - b.EditFileReplaceAll("content/docs/_content.gotmpl", `{{ $.AddPage (dict "kind" "page" "path" "p2" "title" "p2" "dates" $dates "content" $contentHTML ) }}`, "").Build() - b.AssertFileContent("public/index.html", "RegularPagesRecursive: p1:p1:/docs/p1|p1:p1:/docs/p3|pfile:/docs/pfile|$") + b.EditFileReplaceAll("content/docs/_content.gotmpl", `{{ $.AddPage (dict "kind" "page" "path" "p2" "title" "p2title" "dates" $dates "content" $contentHTML ) }}`, "").Build() + b.AssertFileContent("public/index.html", "RegularPagesRecursive: p1:p1:/docs/p1|p3title:/docs/p3|pfile:/docs/pfile|$") } func TestPagesFromGoTmplMovePage(t *testing.T) { @@ -305,6 +307,7 @@ Single: {{ .Title }}|{{ .Content }}| func TestPagesFromGoTmplRemoveGoTmpl(t *testing.T) { t.Parallel() + t.Skip() // TODO1 b := hugolib.TestRunning(t, filesPagesFromDataTempleBasic) b.RemoveFiles("content/docs/_content.gotmpl") b.Build()