Skip to content

Commit

Permalink
Fix multilingual reload when shortcode changes
Browse files Browse the repository at this point in the history 
This commit also refines the partial rebuild logic, to make sure we do not do more work than needed.

Updates gohugoio#2309
  • Loading branch information
@bep @tychoish
bep authored and tychoish committed Aug 13, 2017
1 parent 032d636 commit 69fd52e
Show file tree
Hide file tree
Showing 6 changed files with 135 additions and 74 deletions.
10 changes: 10 additions & 0 deletions hugolib/handler_page.go
View file
Expand Up @@ -15,6 +15,7 @@ package hugolib

import (
"bytes"
"fmt"

"github.com/spf13/hugo/helpers"
"github.com/spf13/hugo/source"
Expand Down Expand Up @@ -67,6 +68,10 @@ type htmlHandler struct {

func (h htmlHandler) Extensions() []string { return []string{"html", "htm"} }
func (h htmlHandler) PageConvert(p *Page, t tpl.Template) HandledResult {
if p.rendered {
panic(fmt.Sprintf("Page %q already rendered, does not need conversion", p.BaseFileName()))
}

p.ProcessShortcodes(t)

return HandledResult{err: nil}
Expand Down Expand Up @@ -100,6 +105,11 @@ func (h mmarkHandler) PageConvert(p *Page, t tpl.Template) HandledResult {
}

func commonConvert(p *Page, t tpl.Template) HandledResult {

if p.rendered {
panic(fmt.Sprintf("Page %q already rendered, does not need conversion", p.BaseFileName()))
}

p.ProcessShortcodes(t)

// TODO(bep) these page handlers need to be re-evaluated, as it is hard to
Expand Down
59 changes: 46 additions & 13 deletions hugolib/hugo_sites.go
View file
Expand Up @@ -220,7 +220,7 @@ func (h *HugoSites) Build(config BuildCfg) error {
}
}

if err := h.preRender(); err != nil {
if err := h.preRender(config, whatChanged{source: true, other: true}); err != nil {
return err
}

Expand Down Expand Up @@ -261,6 +261,10 @@ func (h *HugoSites) Rebuild(config BuildCfg, events ...fsnotify.Event) error {
return errors.New("Rebuild does not support 'ResetState'. Use Build.")
}

if !config.Watching {
return errors.New("Rebuild called when not in watch mode")
}

h.runMode.Watching = config.Watching

firstSite := h.Sites[0]
Expand All @@ -270,7 +274,7 @@ func (h *HugoSites) Rebuild(config BuildCfg, events ...fsnotify.Event) error {
s.resetBuildState()
}

sourceChanged, err := firstSite.reBuild(events)
changed, err := firstSite.reBuild(events)

if err != nil {
return err
Expand All @@ -279,15 +283,15 @@ func (h *HugoSites) Rebuild(config BuildCfg, events ...fsnotify.Event) error {
// Assign pages to sites per translation.
h.setupTranslations(firstSite)

if sourceChanged {
if changed.source {
for _, s := range h.Sites {
if err := s.postProcess(); err != nil {
return err
}
}
}

if err := h.preRender(); err != nil {
if err := h.preRender(config, changed); err != nil {
return err
}

Expand Down Expand Up @@ -391,7 +395,7 @@ func (h *HugoSites) setupTranslations(master *Site) {
// preRender performs build tasks that need to be done as late as possible.
// Shortcode handling is the main task in here.
// TODO(bep) We need to look at the whole handler-chain construct with he below in mind.
func (h *HugoSites) preRender() error {
func (h *HugoSites) preRender(cfg BuildCfg, changed whatChanged) error {

for _, s := range h.Sites {
if err := s.setCurrentLanguageConfig(); err != nil {
Expand All @@ -416,13 +420,13 @@ func (h *HugoSites) preRender() error {
if err := s.setCurrentLanguageConfig(); err != nil {
return err
}
renderShortcodesForSite(s)
s.preparePagesForRender(cfg, changed)
}

return nil
}

func renderShortcodesForSite(s *Site) {
func (s *Site) preparePagesForRender(cfg BuildCfg, changed whatChanged) {
pageChan := make(chan *Page)
wg := &sync.WaitGroup{}

Expand All @@ -431,14 +435,37 @@ func renderShortcodesForSite(s *Site) {
go func(pages <-chan *Page, wg *sync.WaitGroup) {
defer wg.Done()
for p := range pages {

if !changed.other && p.rendered {
// No need to process it again.
continue
}

// If we got this far it means that this is either a new Page pointer
// or a template or similar has changed so wee need to do a rerendering
// of the shortcodes etc.

// Mark it as rendered
p.rendered = true

// If in watch mode, we need to keep the original so we can
// repeat this process on rebuild.
if cfg.Watching {
p.rawContentCopy = make([]byte, len(p.rawContent))
copy(p.rawContentCopy, p.rawContent)
} else {
// Just reuse the same slice.
p.rawContentCopy = p.rawContent
}

if err := handleShortcodes(p, s.owner.tmpl); err != nil {
jww.ERROR.Printf("Failed to handle shortcodes for page %s: %s", p.BaseFileName(), err)
}

if p.Markup == "markdown" {
tmpContent, tmpTableOfContents := helpers.ExtractTOC(p.rawContent)
tmpContent, tmpTableOfContents := helpers.ExtractTOC(p.rawContentCopy)
p.TableOfContents = helpers.BytesToHTML(tmpTableOfContents)
p.rawContent = tmpContent
p.rawContentCopy = tmpContent
}

if p.Markup != "html" {
Expand All @@ -449,19 +476,25 @@ func renderShortcodesForSite(s *Site) {
if err != nil {
jww.ERROR.Printf("Failed to set use defined summary: %s", err)
} else if summaryContent != nil {
p.rawContent = summaryContent.content
p.rawContentCopy = summaryContent.content
}

p.Content = helpers.BytesToHTML(p.rawContent)
p.rendered = true
p.Content = helpers.BytesToHTML(p.rawContentCopy)

if summaryContent == nil {
p.setAutoSummary()
}

} else {
p.Content = helpers.BytesToHTML(p.rawContentCopy)
}

// no need for this anymore
p.rawContentCopy = nil

//analyze for raw stats
p.analyzePage()

}
}(pageChan, wg)
}
Expand Down Expand Up @@ -490,7 +523,7 @@ func handleShortcodes(p *Page, t tpl.Template) error {
return err
}

p.rawContent, err = replaceShortcodeTokens(p.rawContent, shortcodePlaceholderPrefix, shortcodes)
p.rawContentCopy, err = replaceShortcodeTokens(p.rawContentCopy, shortcodePlaceholderPrefix, shortcodes)

if err != nil {
jww.FATAL.Printf("Failed to replace short code tokens in %s:\n%s", p.BaseFileName(), err.Error())
Expand Down
34 changes: 27 additions & 7 deletions hugolib/hugo_sites_test.go
View file
Expand Up @@ -51,7 +51,7 @@ func doTestMultiSitesMainLangInRoot(t *testing.T, defaultInSubDir bool) {
testCommonResetState()
viper.Set("DefaultContentLanguageInSubdir", defaultInSubDir)

sites := createMultiTestSites(t, multiSiteTomlConfig)
sites := createMultiTestSites(t, multiSiteTOMLConfig)

err := sites.Build(BuildCfg{})

Expand Down Expand Up @@ -166,7 +166,7 @@ func TestMultiSitesBuild(t *testing.T) {
content string
suffix string
}{
{multiSiteTomlConfig, "toml"},
{multiSiteTOMLConfig, "toml"},
{multiSiteYAMLConfig, "yml"},
{multiSiteJSONConfig, "json"},
} {
Expand Down Expand Up @@ -323,8 +323,8 @@ func doTestMultiSitesBuild(t *testing.T, configContent, configSuffix string) {

func TestMultiSitesRebuild(t *testing.T) {
testCommonResetState()
sites := createMultiTestSites(t, multiSiteTomlConfig)
cfg := BuildCfg{}
sites := createMultiTestSites(t, multiSiteTOMLConfig)
cfg := BuildCfg{Watching: true}

err := sites.Build(cfg)

Expand All @@ -350,6 +350,10 @@ func TestMultiSitesRebuild(t *testing.T) {
docFr := readDestination(t, "public/fr/sect/doc1/index.html")
assert.True(t, strings.Contains(docFr, "Bonjour"), "No Bonjour")

// check single page content
assertFileContent(t, "public/fr/sect/doc1/index.html", true, "Single", "Shortcode: Bonjour")
assertFileContent(t, "public/en/sect/doc1-slug/index.html", true, "Single", "Shortcode: Hello")

for i, this := range []struct {
preFunc func(t *testing.T)
events []fsnotify.Event
Expand Down Expand Up @@ -474,6 +478,22 @@ func TestMultiSitesRebuild(t *testing.T) {

},
},
// Change a shortcode
{
func(t *testing.T) {
writeSource(t, "layouts/shortcodes/shortcode.html", "Modified Shortcode: {{ i18n \"hello\" }}")
},
[]fsnotify.Event{
{Name: "layouts/shortcodes/shortcode.html", Op: fsnotify.Write},
},
func(t *testing.T) {
assert.Len(t, enSite.Pages, 4)
assert.Len(t, enSite.AllPages, 10)
assert.Len(t, frSite.Pages, 4)
assertFileContent(t, "public/fr/sect/doc1/index.html", true, "Single", "Modified Shortcode: Salut")
assertFileContent(t, "public/en/sect/doc1-slug/index.html", true, "Single", "Modified Shortcode: Hello")
},
},
} {

if this.preFunc != nil {
Expand Down Expand Up @@ -516,7 +536,7 @@ func assertShouldNotBuild(t *testing.T, sites *HugoSites) {
func TestAddNewLanguage(t *testing.T) {
testCommonResetState()

sites := createMultiTestSites(t, multiSiteTomlConfig)
sites := createMultiTestSites(t, multiSiteTOMLConfig)
cfg := BuildCfg{}

err := sites.Build(cfg)
Expand All @@ -525,7 +545,7 @@ func TestAddNewLanguage(t *testing.T) {
t.Fatalf("Failed to build sites: %s", err)
}

newConfig := multiSiteTomlConfig + `
newConfig := multiSiteTOMLConfig + `
[Languages.sv]
weight = 15
Expand Down Expand Up @@ -573,7 +593,7 @@ title = "Svenska"

}

var multiSiteTomlConfig = `
var multiSiteTOMLConfig = `
DefaultExtension = "html"
baseurl = "http://example.com/blog"
DisableSitemap = false
Expand Down
77 changes: 40 additions & 37 deletions hugolib/page.go
View file
Expand Up @@ -48,28 +48,42 @@ var (
)

type Page struct {
Params map[string]interface{}
Content template.HTML
Summary template.HTML
Aliases []string
Status string
Images []Image
Videos []Video
TableOfContents template.HTML
Truncated bool
Draft bool
PublishDate time.Time
ExpiryDate time.Time
Markup string
translations Pages
extension string
contentType string
renderable bool
Layout string
layoutsCalculated []string
linkTitle string
frontmatter []byte
rawContent []byte
Params map[string]interface{}
Content template.HTML
Summary template.HTML
Aliases []string
Status string
Images []Image
Videos []Video
TableOfContents template.HTML
Truncated bool
Draft bool
PublishDate time.Time
ExpiryDate time.Time
Markup string
translations Pages
extension string
contentType string
renderable bool
Layout string
layoutsCalculated []string
linkTitle string
frontmatter []byte

// rawContent isn't "raw" as in the same as in the content file.
// Hugo cares about memory consumption, so we make changes to it to do
// markdown rendering etc., but it is "raw enough" so we can do rebuilds
// when shortcode changes etc.
rawContent []byte

// When running Hugo in watch mode, we do partial rebuilds and have to make
// a copy of the rawContent to be prepared for rebuilds when shortcodes etc.
// have changed.
rawContentCopy []byte

// state telling if this is a "new page" or if we have rendered it previously.
rendered bool

contentShortCodes map[string]func() (string, error)
shortcodes map[string]shortcode
plain string // TODO should be []byte
Expand All @@ -84,7 +98,6 @@ type Page struct {
Source
Position `json:"-"`
Node
rendered bool
}

type Source struct {
Expand Down Expand Up @@ -220,7 +233,7 @@ var (
// Returns the page as summary and main if a user defined split is provided.
func (p *Page) setUserDefinedSummaryIfProvided() (*summaryContent, error) {

sc := splitUserDefinedSummaryAndContent(p.Markup, p.rawContent)
sc := splitUserDefinedSummaryAndContent(p.Markup, p.rawContentCopy)

if sc == nil {
// No divider found
Expand Down Expand Up @@ -1024,19 +1037,9 @@ func (p *Page) SaveSource() error {
}

func (p *Page) ProcessShortcodes(t tpl.Template) {

// these short codes aren't used until after Page render,
// but processed here to avoid coupling
// TODO(bep) Move this and remove p.contentShortCodes
if !p.rendered {
tmpContent, tmpContentShortCodes, _ := extractAndRenderShortcodes(string(p.rawContent), p, t)
p.rawContent = []byte(tmpContent)
p.contentShortCodes = tmpContentShortCodes
} else {
// shortcode template may have changed, rerender
p.contentShortCodes = renderShortcodes(p.shortcodes, p, t)
}

tmpContent, tmpContentShortCodes, _ := extractAndRenderShortcodes(string(p.rawContent), p, t)
p.rawContent = []byte(tmpContent)
p.contentShortCodes = tmpContentShortCodes
}

func (p *Page) FullFilePath() string {
Expand Down
4 changes: 0 additions & 4 deletions hugolib/shortcode.go
View file
Expand Up @@ -281,10 +281,6 @@ func renderShortcode(sc shortcode, parent *ShortcodeWithPage, p *Page, t tpl.Tem

func extractAndRenderShortcodes(stringToParse string, p *Page, t tpl.Template) (string, map[string]func() (string, error), error) {

if p.rendered {
panic("Illegal state: Page already marked as rendered, please reuse the shortcodes")
}

content, shortcodes, err := extractShortcodes(stringToParse, p, t)

if err != nil {
Expand Down

0 comments on commit 69fd52e

Please sign in to comment.