Skip to content

Commit

Permalink
Tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@bep
bep committed May 11, 2024
1 parent 779e246 commit c069a84
Show file tree
Hide file tree
Showing 6 changed files with 125 additions and 57 deletions.
37 changes: 14 additions & 23 deletions hugolib/content_map.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,7 @@ func (m *pageMap) AddFi(fi hugofs.FileMetaInfo, whatChanged *whatChanged) (pageC
_, n, replaced := s.pageMap.treePages.InsertIntoValuesDimensionWithLock(pi.Base(), ps)

if h.isRebuild() && replaced {
pt.BuildState.ChangedIdentities = append(pt.BuildState.ChangedIdentities, n.GetIdentity())
pt.AddChange(n.GetIdentity())
}

return nil
Expand All @@ -313,7 +313,7 @@ func (m *pageMap) AddFi(fi hugofs.FileMetaInfo, whatChanged *whatChanged) (pageC
_, n, replaced := s.pageMap.treeResources.InsertIntoValuesDimensionWithLock(rc.Path, rs)

if h.isRebuild() && replaced {
pt.BuildState.ChangedIdentities = append(pt.BuildState.ChangedIdentities, n.GetIdentity())
pt.AddChange(n.GetIdentity())
}
return nil
},
Expand All @@ -324,14 +324,20 @@ func (m *pageMap) AddFi(fi hugofs.FileMetaInfo, whatChanged *whatChanged) (pageC

}

contentAdapter.BuildState.PrepareNextBuild()
handleBuildInfo := func(s *Site, bi pagesfromdata.BuildInfo) {
resourceCount += bi.NumResourcesAdded
pageCount += bi.NumPagesAdded
s.handleContentAdapterChanges(bi, whatChanged)
}

// TODO1 do we need a mutex?
if err := contentAdapter.Execute(context.Background()); err != nil {
bi, err := contentAdapter.Execute(context.Background())
if err != nil {
return err
}
handleBuildInfo(s, bi)

if !rebuild && contentAdapter.BuildState.EnableAllLanguages {
if !rebuild && bi.EnableAllLanguages {
// Clone and insert the adapter for the other sites.
for _, ss := range s.h.Sites {
if s == ss {
Expand All @@ -341,33 +347,18 @@ func (m *pageMap) AddFi(fi hugofs.FileMetaInfo, whatChanged *whatChanged) (pageC
clone := contentAdapter.CloneForSite(ss)

// Make sure it gets executed for the first time.
if err := clone.Execute(context.Background()); err != nil {
bi, err := clone.Execute(context.Background())
if err != nil {
return err
}
handleBuildInfo(ss, bi)

// Insert into the correct language tree so it get rebuilt on changes.
ss.pageMap.treePagesFromTemplateOptions.Insert(pi.Base(), clone)

}
}

resourceCount += contentAdapter.BuildState.NumResourcesAdded
pageCount += contentAdapter.BuildState.NumPagesAdded

if m.s.h.isRebuild() {
if len(contentAdapter.BuildState.ChangedIdentities) > 0 {
whatChanged.Add(contentAdapter.BuildState.ChangedIdentities...)
}

for _, p := range contentAdapter.BuildState.DeletedPaths {
// TODO1 language, resource etc.
pp := path.Join(pi.Base(), p)
if v, ok := m.treePages.Delete(pp); ok {
whatChanged.Add(v.GetIdentity())
}

}
}
return nil
}(); err != nil {
addErr = err
Expand Down
2 changes: 1 addition & 1 deletion hugolib/content_map_page.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1151,7 +1151,7 @@ func (h *HugoSites) resolveAndClearStateForIdentities(
for _, id := range changes {
checkedCount++
if r := depsFinder.Contains(id, n.DependencyManager, 2); r > identity.FinderNotFound {
n.BuildState.Rebuild = true
n.MarkStale()
matchCount++
break
}
Expand Down
7 changes: 5 additions & 2 deletions hugolib/doctree/treeshifttree.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,10 +55,13 @@ func (t *TreeShiftTree[T]) Get(s string) T {
return t.trees[t.v].Get(s)
}

func (t *TreeShiftTree[T]) GetAll(s string, f func(v T)) {
func (t *TreeShiftTree[T]) DeleteAllFunc(s string, f func(s string, v T) bool) {
for _, tt := range t.trees {
if v := tt.Get(s); v != t.zero {
f(v)
if f(s, v) {
// Delete.
tt.tree.Delete(s)
}
}
}
}
Expand Down
51 changes: 45 additions & 6 deletions hugolib/hugo_sites_build.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,14 @@ func (h *HugoSites) assemble(ctx context.Context, l logg.LevelLogger, bcfg *Buil
defer loggers.TimeTrackf(l, time.Now(), nil, "")

if !bcfg.whatChanged.contentChanged {
changes := bcfg.whatChanged.Changes()

if len(changes) > 0 {
// TODO1 consolidate.
if err := h.resolveAndClearStateForIdentities(ctx, l, nil, changes); err != nil {
return err
}
}
return nil
}

Expand Down Expand Up @@ -709,9 +717,17 @@ func (h *HugoSites) processPartial(ctx context.Context, l logg.LevelLogger, conf
changes = append(changes, ids...)
}
} else {
h.pageTrees.treePagesFromTemplateOptions.GetAll(pathInfo.Base(),
func(n *pagesfromdata.PagesFromTemplate) {
h.pageTrees.treePagesFromTemplateOptions.DeleteAllFunc(pathInfo.Base(),
func(s string, n *pagesfromdata.PagesFromTemplate) bool {
changes = append(changes, n.DependencyManager)

// Try to open the file to see if has been deleted.
f, err := n.GoTmplFi.Meta().Open()
if err == nil {
f.Close()
}
// TODO1 also remove all pages and resources below this path.
return err != nil
})
}

Expand Down Expand Up @@ -928,7 +944,7 @@ func (h *HugoSites) processPartial(ctx context.Context, l logg.LevelLogger, conf
}

if h.isRebuild() {
if err := h.processContentAdaptersOnRebuild(ctx); err != nil {
if err := h.processContentAdaptersOnRebuild(ctx, config); err != nil {
return err
}
}
Expand All @@ -955,16 +971,39 @@ func (h *HugoSites) processFull(ctx context.Context, l logg.LevelLogger, config
return err
}

func (h *HugoSites) processContentAdaptersOnRebuild(ctx context.Context) error {
func (s *Site) handleContentAdapterChanges(bi pagesfromdata.BuildInfo, whatChanged *whatChanged) {
if !s.h.isRebuild() {
return
}

if len(bi.ChangedIdentities) > 0 {
whatChanged.Add(bi.ChangedIdentities...)
}

for _, p := range bi.DeletedPaths {
pp := path.Join(bi.Path.Base(), p)
if v, ok := s.pageMap.treePages.Delete(pp); ok {
whatChanged.Add(v.GetIdentity())
}
}
}

func (h *HugoSites) processContentAdaptersOnRebuild(ctx context.Context, buildConfig *BuildCfg) error {
g := rungroup.Run[*pagesfromdata.PagesFromTemplate](ctx, rungroup.Config[*pagesfromdata.PagesFromTemplate]{
NumWorkers: h.numWorkers,
Handle: func(ctx context.Context, p *pagesfromdata.PagesFromTemplate) error {
return p.Execute(ctx)
bi, err := p.Execute(ctx)
if err != nil {
return err
}
s := p.Site.(*Site)
s.handleContentAdapterChanges(bi, buildConfig.whatChanged)
return nil
},
})

h.pageTrees.treePagesFromTemplateOptions.WalkPrefixRaw(doctree.LockTypeRead, "", func(key string, p *pagesfromdata.PagesFromTemplate) (bool, error) {
if p.BuildState.Rebuild {
if p.StaleVersion() > 0 {
g.Enqueue(p)
}
return false, nil
Expand Down
72 changes: 52 additions & 20 deletions hugolib/pagesfromdata/pagesfromgotmpl.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,13 @@ import (
"path/filepath"

"github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/common/paths"
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/hugofs"
"github.com/gohugoio/hugo/identity"
"github.com/gohugoio/hugo/resources/page"
"github.com/gohugoio/hugo/resources/page/pagemeta"
"github.com/gohugoio/hugo/resources/resource"
"github.com/gohugoio/hugo/tpl"
"github.com/mitchellh/mapstructure"
"github.com/spf13/cast"
Expand Down Expand Up @@ -81,7 +83,7 @@ func (p *pagesFromDataTemplateContext) AddPage(v any) (string, error) {
return "", err
}

if !p.p.BuildState.checkHasChangedAndSetSourceInfo(path, m) {
if !p.p.buildState.checkHasChangedAndSetSourceInfo(path, m) {
return "", nil
}

Expand All @@ -96,7 +98,7 @@ func (p *pagesFromDataTemplateContext) AddPage(v any) (string, error) {
// TODO1
pd.Build = pagemeta.DefaultBuildConfig

p.p.BuildState.NumPagesAdded++
p.p.buildState.NumPagesAdded++

if err := pd.Validate(true); err != nil {
return "", err
Expand All @@ -111,7 +113,7 @@ func (p *pagesFromDataTemplateContext) AddResource(v any) (string, error) {
return "", err
}

if !p.p.BuildState.checkHasChangedAndSetSourceInfo(path, m) {
if !p.p.buildState.checkHasChangedAndSetSourceInfo(path, m) {
return "", nil
}

Expand All @@ -120,7 +122,7 @@ func (p *pagesFromDataTemplateContext) AddResource(v any) (string, error) {
return "", err
}

p.p.BuildState.NumResourcesAdded++
p.p.buildState.NumResourcesAdded++

if err := rd.Validate(); err != nil {
return "", err
Expand All @@ -138,15 +140,15 @@ func (p *pagesFromDataTemplateContext) Store() *maps.Scratch {
}

func (p *pagesFromDataTemplateContext) EnableAllLanguages() string {
p.p.BuildState.EnableAllLanguages = true
p.p.buildState.EnableAllLanguages = true
return ""
}

func NewPagesFromTemplate(opts PagesFromTemplateOptions) *PagesFromTemplate {
return &PagesFromTemplate{
PagesFromTemplateOptions: opts,
PagesFromTemplateDeps: opts.DepsFromSite(opts.Site),
BuildState: &BuildState{
buildState: &BuildState{
sourceInfosCurrent: maps.NewCache[string, *sourceInfo](),
},
store: maps.NewScratch(),
Expand All @@ -172,15 +174,38 @@ type PagesFromTemplateDeps struct {
TmplExec tpl.TemplateExecutor
}

var _ resource.Staler = (*PagesFromTemplate)(nil)

type PagesFromTemplate struct {
PagesFromTemplateOptions
PagesFromTemplateDeps
BuildState *BuildState
buildState *BuildState
store *maps.Scratch
}

func (b *PagesFromTemplate) AddChange(id identity.Identity) {
b.buildState.ChangedIdentities = append(b.buildState.ChangedIdentities, id)
}

func (b *PagesFromTemplate) MarkStale() {
b.buildState.StaleVersion++
}

func (b *PagesFromTemplate) StaleVersion() uint32 {
return b.buildState.StaleVersion
}

type BuildInfo struct {
NumPagesAdded uint64
NumResourcesAdded uint64
EnableAllLanguages bool
ChangedIdentities []identity.Identity
DeletedPaths []string
Path *paths.Path
}

type BuildState struct {
Rebuild bool
StaleVersion uint32

EnableAllLanguages bool

Expand Down Expand Up @@ -249,12 +274,10 @@ func (b *BuildState) resolveDeletedPaths() {
b.DeletedPaths = paths
}

func (b *BuildState) prepareNextExecute() {
func (b *BuildState) PrepareNextBuild() {
b.sourceInfosPrevious = b.sourceInfosCurrent
b.sourceInfosCurrent = maps.NewCache[string, *sourceInfo]()
}

func (b *BuildState) PrepareNextBuild() {
b.StaleVersion = 0
b.DeletedPaths = nil
b.ChangedIdentities = nil
b.NumPagesAdded = 0
Expand All @@ -269,7 +292,7 @@ func (p PagesFromTemplate) CloneForSite(s page.Site) *PagesFromTemplate {
// We deliberately make them share the same DepenencyManager and Store.
p.PagesFromTemplateOptions.Site = s
p.PagesFromTemplateDeps = p.PagesFromTemplateOptions.DepsFromSite(s)
p.BuildState = &BuildState{
p.buildState = &BuildState{
sourceInfosCurrent: maps.NewCache[string, *sourceInfo](),
}
return &p
Expand All @@ -284,20 +307,20 @@ func (p *PagesFromTemplate) GetDependencyManagerForScope(scope int) identity.Man
return p.DependencyManager
}

func (p *PagesFromTemplate) Execute(ctx context.Context) error {
func (p *PagesFromTemplate) Execute(ctx context.Context) (BuildInfo, error) {
defer func() {
p.BuildState.prepareNextExecute()
p.buildState.PrepareNextBuild()
}()

f, err := p.GoTmplFi.Meta().Open()
if err != nil {
return err
return BuildInfo{}, err
}
defer f.Close()

tmpl, err := p.TmplFinder.Parse(filepath.ToSlash(p.GoTmplFi.Meta().Filename), helpers.ReaderToString(f))
if err != nil {
return err
return BuildInfo{}, err
}

data := &pagesFromDataTemplateContext{
Expand All @@ -307,13 +330,22 @@ func (p *PagesFromTemplate) Execute(ctx context.Context) error {
ctx = tpl.Context.DependencyManagerScopedProvider.Set(ctx, p)

if err := p.TmplExec.ExecuteWithContext(ctx, tmpl, io.Discard, data); err != nil {
return err
return BuildInfo{}, err
}

p.BuildState.resolveDeletedPaths()
p.buildState.resolveDeletedPaths()
// p.BuildState.printDebug()

return nil
bi := BuildInfo{
NumPagesAdded: p.buildState.NumPagesAdded,
NumResourcesAdded: p.buildState.NumResourcesAdded,
EnableAllLanguages: p.buildState.EnableAllLanguages,
ChangedIdentities: p.buildState.ChangedIdentities,
DeletedPaths: p.buildState.DeletedPaths,
Path: p.GoTmplFi.Meta().PathInfo,
}

return bi, nil
}

//////////////

0 comments on commit c069a84

Please sign in to comment.