diff --git a/helpers/content.go b/helpers/content.go index e955b0004c9..b9281acf406 100644 --- a/helpers/content.go +++ b/helpers/content.go @@ -41,6 +41,8 @@ var SummaryLength = 70 // SummaryDivider denotes where content summarization should end. The default is "". var SummaryDivider = []byte("") +var summaryDividerAndNewLines = []byte("\n\n") + // Blackfriday holds configuration values for Blackfriday rendering. type Blackfriday struct { Smartypants bool @@ -390,8 +392,21 @@ func WordCount(s string) map[string]int { } // RemoveSummaryDivider removes summary-divider from content. +// TODO(bep) ml remove func RemoveSummaryDivider(content []byte) []byte { - return bytes.Replace(content, SummaryDivider, []byte(""), -1) + b := bytes.Replace(content, summaryDividerAndNewLines, []byte(""), 1) + if len(b) != len(content) { + return b + } + return bytes.Replace(content, SummaryDivider, []byte(""), 1) +} + +func removeInternalSummaryDivider(content []byte) []byte { + b := bytes.Replace(content, summaryDividerAndNewLines, []byte(""), 1) + if len(b) != len(content) { + return b + } + return bytes.Replace(content, SummaryDivider, []byte(""), 1) } // TruncateWordsByRune truncates words by runes. diff --git a/hugolib/handler_meta.go b/hugolib/handler_meta.go index 0825ef1e68f..36b0f5a2a10 100644 --- a/hugolib/handler_meta.go +++ b/hugolib/handler_meta.go @@ -75,8 +75,6 @@ func (mh *MetaHandle) Convert(i interface{}, s *Site, results HandleResults) { } results <- h.PageConvert(p, s.Tmpl) - p.setSummary() - p.analyzePage() } } diff --git a/hugolib/handler_page.go b/hugolib/handler_page.go index 73a972446ce..04af20ceb76 100644 --- a/hugolib/handler_page.go +++ b/hugolib/handler_page.go @@ -14,10 +14,11 @@ package hugolib import ( + "bytes" + "github.com/spf13/hugo/helpers" "github.com/spf13/hugo/source" "github.com/spf13/hugo/tpl" - jww "github.com/spf13/jwalterweatherman" "github.com/spf13/viper" ) @@ -67,18 +68,7 @@ type htmlHandler struct { func (h htmlHandler) Extensions() []string { return []string{"html", "htm"} } func (h htmlHandler) PageConvert(p *Page, t tpl.Template) HandledResult { p.ProcessShortcodes(t) - var err error - - if len(p.contentShortCodes) > 0 { - p.rawContent, err = replaceShortcodeTokens(p.rawContent, shortcodePlaceholderPrefix, p.contentShortCodes) - if err != nil { - jww.FATAL.Printf("Failed to replace short code tokens in %s:\n%s", p.BaseFileName(), err.Error()) - return HandledResult{err: err} - } - } - - p.Content = helpers.BytesToHTML(p.rawContent) return HandledResult{err: nil} } @@ -112,27 +102,22 @@ func (h mmarkHandler) PageConvert(p *Page, t tpl.Template) HandledResult { func commonConvert(p *Page, t tpl.Template) HandledResult { p.ProcessShortcodes(t) - var err error - + // TODO(bep) these page handlers need to be re-evaluated, as it is hard to + // process a page in isolation. See the new preRender func. + // TODO(bep) ml not so raw anymore, but do we need to keep it raw? if viper.GetBool("EnableEmoji") { p.rawContent = helpers.Emojify(p.rawContent) } - renderedContent := p.renderContent(helpers.RemoveSummaryDivider(p.rawContent)) - - if len(p.contentShortCodes) > 0 { - renderedContent, err = replaceShortcodeTokens(renderedContent, shortcodePlaceholderPrefix, p.contentShortCodes) - - if err != nil { - jww.FATAL.Printf("Failed to replace shortcode tokens in %s:\n%s", p.BaseFileName(), err.Error()) - return HandledResult{err: err} - } - } - - tmpContent, tmpTableOfContents := helpers.ExtractTOC(renderedContent) + // TODO(bep) ml we let the summary divider survive the rendering. Must check if + // it actually survives, replace it with something more robus, or maybe + // rethink this fragile concept. + //p.rawContent = p.renderContent(helpers.RemoveSummaryDivider(p.rawContent)) + // We have to replace the with something that survives all the + // rendering engines. + // TODO(bep) inline replace + p.rawContent = bytes.Replace(p.rawContent, []byte(helpers.SummaryDivider), internalSummaryDivider, 1) + p.rawContent = p.renderContent(p.rawContent) - p.Content = helpers.BytesToHTML(tmpContent) - p.TableOfContents = helpers.BytesToHTML(tmpTableOfContents) - p.rendered = true return HandledResult{err: nil} } diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index 2dd1bb9bef8..76f9979fd90 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -16,8 +16,11 @@ package hugolib import ( "errors" "strings" + "sync" "time" + "github.com/spf13/hugo/helpers" + "github.com/spf13/viper" "github.com/fsnotify/fsnotify" @@ -106,24 +109,27 @@ func (h HugoSites) Build(config BuildCfg) error { } for _, s := range h.Sites { - if err := s.PostProcess(); err != nil { return err } + } + + if err := h.preRender(); err != nil { + return err + } + + for _, s := range h.Sites { if !config.skipRender { if err := s.Render(); err != nil { return err } + if config.PrintStats { + s.Stats() + } } - - if config.PrintStats { - s.Stats() - } - // TODO(bep) ml lang in site.Info? - // TODO(bep) ml Page sorting? } if config.PrintStats { @@ -153,22 +159,26 @@ func (h HugoSites) Rebuild(config BuildCfg, events ...fsnotify.Event) error { // Assign pages to sites per translation. h.setupTranslations(firstSite) - for _, s := range h.Sites { - - if sourceChanged { + if sourceChanged { + for _, s := range h.Sites { if err := s.PostProcess(); err != nil { return err } } + } - if !config.skipRender { + if err := h.preRender(); err != nil { + return err + } + + if !config.skipRender { + for _, s := range h.Sites { if err := s.Render(); err != nil { return err } - } - - if config.PrintStats { - s.Stats() + if config.PrintStats { + s.Stats() + } } } @@ -219,6 +229,87 @@ func (s *HugoSites) setupTranslations(master *Site) { } } +// preRender performs build tasks that needs to be done as late as possible. +// Shortcode handling is the main task in here. +// TODO(bep) We need to look at the whole handler-chain construct witht he below in mind. +func (h *HugoSites) preRender() error { + pageChan := make(chan *Page) + + wg := &sync.WaitGroup{} + + // We want all the pages, so just pick one. + s := h.Sites[0] + + for i := 0; i < getGoMaxProcs()*4; i++ { + wg.Add(1) + go func(pages <-chan *Page, wg *sync.WaitGroup) { + defer wg.Done() + for p := range pages { + if err := handleShortcodes(p, s.Tmpl); err != nil { + jww.ERROR.Printf("Failed to handle shortcodes for page %s: %s", p.BaseFileName(), err) + } + + if p.Markup == "markdown" { + tmpContent, tmpTableOfContents := helpers.ExtractTOC(p.rawContent) + p.TableOfContents = helpers.BytesToHTML(tmpTableOfContents) + p.rawContent = tmpContent + } + + if p.Markup != "html" { + + // Now we know enough to create a summary of the page and count some words + summaryContent, err := p.setUserDefinedSummaryIfProvided() + + if err != nil { + jww.ERROR.Printf("Failed to set use defined summary: %s", err) + } else if summaryContent != nil { + p.rawContent = summaryContent.content + } + + p.Content = helpers.BytesToHTML(p.rawContent) + p.rendered = true + + if summaryContent == nil { + p.setAutoSummary() + } + } + + //analyze for raw stats + p.analyzePage() + } + }(pageChan, wg) + } + + for _, p := range s.AllPages { + pageChan <- p + } + + close(pageChan) + + wg.Wait() + + return nil +} + +func handleShortcodes(p *Page, t tpl.Template) error { + if len(p.contentShortCodes) > 0 { + jww.DEBUG.Printf("Replace %d shortcodes in %q", len(p.contentShortCodes), p.BaseFileName()) + shortcodes, err := executeShortcodeFuncMap(p.contentShortCodes) + + if err != nil { + return err + } + + p.rawContent, err = replaceShortcodeTokens(p.rawContent, shortcodePlaceholderPrefix, shortcodes) + + if err != nil { + jww.FATAL.Printf("Failed to replace short code tokens in %s:\n%s", p.BaseFileName(), err.Error()) + } + } + + return nil +} + func (s *Site) updateBuildStats(page *Page) { if page.IsDraft() { s.draftCount++ diff --git a/hugolib/hugo_sites_test.go b/hugolib/hugo_sites_test.go index fc48011159b..b63792a3bc7 100644 --- a/hugolib/hugo_sites_test.go +++ b/hugolib/hugo_sites_test.go @@ -37,6 +37,11 @@ func testCommonResetState() { viper.Set("PublishDir", "public") viper.Set("RSSUri", "rss") + viper.Set("Taxonomies", map[string]interface{}{ + "tag": "tags", + "category": "categories", + }) + if err := hugofs.Source().Mkdir("content", 0755); err != nil { panic("Content folder creation failed.") } diff --git a/hugolib/page.go b/hugolib/page.go index 4248ff893ae..493e5b51228 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -70,7 +70,7 @@ type Page struct { linkTitle string frontmatter []byte rawContent []byte - contentShortCodes map[string]string // TODO(bep) this shouldn't be needed. + contentShortCodes map[string]func() (string, error) shortcodes map[string]shortcode plain string // TODO should be []byte plainWords []string @@ -212,46 +212,111 @@ func (p *Page) lineNumRawContentStart() int { return bytes.Count(p.frontmatter, []byte("\n")) + 1 } -func (p *Page) setSummary() { +var ( + internalSummaryDivider = []byte("HUGOMORE42") + internalSummaryDividerAndNewLines = []byte("HUGOMORE42\n\n") +) - // at this point, p.rawContent contains placeholders for the short codes, - // rendered and ready in p.contentShortcodes +// Returns the page as summary and main if a user defined split is provided. +func (p *Page) setUserDefinedSummaryIfProvided() (*summaryContent, error) { - if bytes.Contains(p.rawContent, helpers.SummaryDivider) { - sections := bytes.Split(p.rawContent, helpers.SummaryDivider) - header := sections[0] - p.Truncated = true - if len(sections[1]) < 20 { - // only whitespace? - p.Truncated = len(bytes.Trim(sections[1], " \n\r")) > 0 - } + sc := splitUserDefinedSummaryAndContent(p.Markup, p.rawContent) - // TODO(bep) consider doing this once only - renderedHeader := p.renderBytes(header) - if len(p.contentShortCodes) > 0 { - tmpContentWithTokensReplaced, err := - replaceShortcodeTokens(renderedHeader, shortcodePlaceholderPrefix, p.contentShortCodes) - if err != nil { - jww.FATAL.Printf("Failed to replace short code tokens in Summary for %s:\n%s", p.BaseFileName(), err.Error()) - } else { - renderedHeader = tmpContentWithTokensReplaced - } - } - p.Summary = helpers.BytesToHTML(renderedHeader) - } else { - // If hugo defines split: - // render, strip html, then split - var summary string - var truncated bool - if p.isCJKLanguage { - summary, truncated = helpers.TruncateWordsByRune(p.PlainWords(), helpers.SummaryLength) - } else { - summary, truncated = helpers.TruncateWordsToWholeSentence(p.PlainWords(), helpers.SummaryLength) - } - p.Summary = template.HTML(summary) - p.Truncated = truncated + if sc == nil { + // No divider found + return nil, nil + } + p.Truncated = true + if len(sc.content) < 20 { + // only whitespace? + p.Truncated = len(bytes.Trim(sc.content, " \n\r")) > 0 } + + p.Summary = helpers.BytesToHTML(sc.summary) + + return sc, nil +} + +// Make this explicit so there is no doubt about what is what. +type summaryContent struct { + summary []byte + content []byte + contentWithoutSummary []byte +} + +func splitUserDefinedSummaryAndContent(markup string, c []byte) *summaryContent { + startDivider := bytes.Index(c, internalSummaryDivider) + + if startDivider == -1 { + return nil + } + + endDivider := startDivider + len(internalSummaryDivider) + endSummary := startDivider + + var ( + startMarkup []byte + endMarkup []byte + addDiv bool + divStart = []byte("
") + endMarkup = []byte("
") + case "asciidoc": + startMarkup = []byte("") + endMarkup = []byte("
") + addDiv = true + } + + // Find the closest end/start markup string to the divider + //firstStart := bytes.Index(c[:startDivider], startMarkup) + fromIdx := bytes.LastIndex(c[:startDivider], startMarkup) + fromStart := startDivider - fromIdx - len(startMarkup) + fromEnd := bytes.Index(c[endDivider:], endMarkup) + + if fromEnd != -1 && fromEnd <= fromStart { + endSummary = startDivider + fromEnd + len(endMarkup) + } else if fromStart != -1 { + endSummary = startDivider - fromStart - len(startMarkup) + } + + withoutDivider := bytes.TrimSpace(append(c[:startDivider], c[endDivider:]...)) + contentWithoutSummary := bytes.TrimSpace(withoutDivider[endSummary:]) + summary := bytes.TrimSpace(withoutDivider[:endSummary]) + + if addDiv { + // For the rst + summary = append(append([]byte(nil), summary...), []byte("Simple Page
\n")) + checkPageSummary(t, p, "Simple Page") + checkPageType(t, p, "page") + checkPageLayout(t, p, "page/single.html", "_default/single.html", "theme/page/single.html", "theme/_default/single.html") + checkTruncation(t, p, false, "simple short page") } - assert.False(t, p.IsHome) - checkPageTitle(t, p, "Simple") - checkPageContent(t, p, "Simple Page
\n") - checkPageSummary(t, p, "Simple Page") - checkPageType(t, p, "page") - checkPageLayout(t, p, "page/single.html", "_default/single.html", "theme/page/single.html", "theme/_default/single.html") - checkTruncation(t, p, false, "simple short page") + testAllMarkdownEnginesForPage(t, assertFunc, "simple", simplePage) +} + +func TestSplitSummaryAndContent(t *testing.T) { + + for i, this := range []struct { + markup string + content string + expectedSummary string + expectedContent string + expectedContentWithoutSummary string + }{ + {"markdown", `Summary Same LineHUGOMORE42
+ +Some more text
`, "Summary Same Line
", "Summary Same Line
\n\nSome more text
", "Some more text
"}, + {"asciidoc", `sn
HUGOMORE42Some more text
sn
sn
Some more text
Some more text
Summary Next Line
HUGOMORE42Some more text
Summary Next Line
Summary Next Line
Some more text
Some more text
a
b
HUGOMORE42c
", "a
b
", "a
b
c
", "c
"}, + {"markdown", "a
b
cHUGOMORE42
", "a
b
c
", "a
b
c
", ""}, + {"markdown", "a
bHUGOMORE42
c
", "a
b
", "a
b
c
", "c
"}, + {"markdown", "aHUGOMORE42
b
c
", "a
", "a
b
c
", "b
c
"}, + } { + + sc := splitUserDefinedSummaryAndContent(this.markup, []byte(this.content)) + + require.NotNil(t, sc, fmt.Sprintf("[%d] Nil %s", i, this.markup)) + require.Equal(t, this.expectedSummary, string(sc.summary), fmt.Sprintf("[%d] Summary markup %s", i, this.markup)) + require.Equal(t, this.expectedContent, string(sc.content), fmt.Sprintf("[%d] Content markup %s", i, this.markup)) + require.Equal(t, this.expectedContentWithoutSummary, string(sc.contentWithoutSummary), fmt.Sprintf("[%d] Content without summary, markup %s", i, this.markup)) + } + + if true { + return + } + + ad := `sn
HUGOMORE42 +Some more text
+Summary Same LineHUGOMORE42
+ +Some more text
` + + sc := splitUserDefinedSummaryAndContent("markdown", []byte(md)) + + require.Equal(t, "adf", string(sc.summary)) + require.Equal(t, "asdf", string(sc.content)) + + if true { + return + } + sc = splitUserDefinedSummaryAndContent("asciidoc", []byte(ad)) + require.Equal(t, "sn
\nSome more text
\nSummary Next Line
\n\nSome more text
\n"), ext) + checkPageSummary(t, p, normalizeExpected(ext, "Summary Next Line
"), ext) + checkPageType(t, p, "page") + checkPageLayout(t, p, "page/single.html", "_default/single.html", "theme/page/single.html", "theme/_default/single.html") + checkTruncation(t, p, true, "page with summary delimiter") + } + + testAllMarkdownEnginesForPage(t, assertFunc, "simple", simplePageWithSummaryDelimiter) +} + +// Issue #1076 +func TestPageWithDelimiterForMarkdownThatCrossesBorder(t *testing.T) { + s := newSiteFromSources("simple.md", simplePageWithSummaryDelimiterAndMarkdownThatCrossesBorder) + + if err := buildSiteSkipRender(s); err != nil { + t.Fatalf("Failed to build site: %s", err) + } + + require.Len(t, s.Pages, 1) + + p := s.Pages[0] + + if p.Summary != template.HTML("The best static site generator.1\n
") { + t.Fatalf("Got summary:\n%q", p.Summary) + } + + if p.Content != template.HTML("The best static site generator.1\n
\nSummary Next Line
\n\nSome more text
\n") - checkPageSummary(t, p, "Summary Next Line
\n") - checkPageType(t, p, "page") - checkPageLayout(t, p, "page/single.html", "_default/single.html", "theme/page/single.html", "theme/_default/single.html") - checkTruncation(t, p, true, "page with summary delimiter") } func TestPageWithShortCodeInSummary(t *testing.T) { - s := new(Site) - s.prepTemplates(nil) - p, _ := NewPage("simple.md") - _, err := p.ReadFrom(strings.NewReader(simplePageWithShortcodeInSummary)) - if err != nil { - t.Fatalf("Unable to create a page with frontmatter and body content: %s", err) + + assertFunc := func(t *testing.T, ext string, p *Page) { + checkPageTitle(t, p, "Simple") + checkPageContent(t, p, normalizeExpected(ext, "Summary Next Line. .\nMore text here.
Some more text
"), ext) + checkPageSummary(t, p, "Summary Next Line. . More text here. Some more text", ext) + checkPageType(t, p, "page") + checkPageLayout(t, p, "page/single.html", "_default/single.html", "theme/page/single.html", "theme/_default/single.html") } - p.Convert() - checkPageTitle(t, p, "Simple") - checkPageContent(t, p, "Summary Next Line. \n\n.\nMore text here.
\n\nSome more text
\n") - checkPageSummary(t, p, "Summary Next Line. . More text here. Some more text") - checkPageType(t, p, "page") - checkPageLayout(t, p, "page/single.html", "_default/single.html", "theme/page/single.html", "theme/_default/single.html") + testAllMarkdownEnginesForPage(t, assertFunc, "simple", simplePageWithShortcodeInSummary) } func TestPageWithEmbeddedScriptTag(t *testing.T) { - p, _ := NewPage("simple.md") - _, err := p.ReadFrom(strings.NewReader(simplePageWithEmbeddedScript)) - p.Convert() - if err != nil { - t.Fatalf("Unable to create a page with frontmatter and body content: %s", err) + + assertFunc := func(t *testing.T, ext string, p *Page) { + if ext == "ad" || ext == "rst" { + // TOD(bep) + return + } + checkPageContent(t, p, "\n", ext) } - checkPageContent(t, p, "\n") + + testAllMarkdownEnginesForPage(t, assertFunc, "simple", simplePageWithEmbeddedScript) } func TestPageWithAdditionalExtension(t *testing.T) { - p, _ := NewPage("simple.md") - _, err := p.ReadFrom(strings.NewReader(simplePageWithAdditionalExtension)) - p.Convert() - if err != nil { - t.Fatalf("Unable to create a page with frontmatter and body content: %s", err) + s := newSiteFromSources("simple.md", simplePageWithAdditionalExtension) + + if err := buildSiteSkipRender(s); err != nil { + t.Fatalf("Failed to build site: %s", err) } + + require.Len(t, s.Pages, 1) + + p := s.Pages[0] + checkPageContent(t, p, "first line.
\nsecond line.
fourth line.
\n") } func TestTableOfContents(t *testing.T) { - p, _ := NewPage("tocpage.md") - _, err := p.ReadFrom(strings.NewReader(pageWithToC)) - p.Convert() - if err != nil { - t.Fatalf("Unable to create a page with frontmatter and body content: %s", err) + s := newSiteFromSources("tocpage.md", pageWithToC) + + if err := buildSiteSkipRender(s); err != nil { + t.Fatalf("Failed to build site: %s", err) } + + require.Len(t, s.Pages, 1) + + p := s.Pages[0] + checkPageContent(t, p, "\n\nFor some moments the old man did not reply. He stood with bowed head, buried in deep thought. But at last he spoke.
\n\nI have no idea, of course, how long it took me to reach the limit of the plain,\nbut at last I entered the foothills, following a pretty little canyon upward\ntoward the mountains. Beside me frolicked a laughing brooklet, hurrying upon\nits noisy way down to the silent sea. In its quieter pools I discovered many\nsmall fish, of four-or five-pound weight I should imagine. In appearance,\nexcept as to size and color, they were not unlike the whale of our own seas. As\nI watched them playing about I discovered, not only that they suckled their\nyoung, but that at intervals they rose to the surface to breathe as well as to\nfeed upon certain grasses and a strange, scarlet lichen which grew upon the\nrocks just above the water line.
\n\nI remember I felt an extraordinary persuasion that I was being played with,\nthat presently, when I was upon the very verge of safety, this mysterious\ndeath–as swift as the passage of light–would leap after me from the pit about\nthe cylinder and strike me down. ## BB
\n\n“You’re a great Granser,” he cried delightedly, “always making believe them little marks mean something.”
\n") checkPageTOC(t, p, "") } func TestPageWithMoreTag(t *testing.T) { - p, _ := NewPage("simple.md") - _, err := p.ReadFrom(strings.NewReader(simplePageWithSummaryDelimiterSameLine)) - p.Convert() - if err != nil { - t.Fatalf("Unable to create a page with frontmatter and body content: %s", err) + + assertFunc := func(t *testing.T, ext string, p *Page) { + checkPageTitle(t, p, "Simple") + checkPageContent(t, p, normalizeExpected(ext, "Summary Same Line
\n\nSome more text
\n")) + checkPageSummary(t, p, normalizeExpected(ext, "Summary Same Line
")) + checkPageType(t, p, "page") + checkPageLayout(t, p, "page/single.html", "_default/single.html", "theme/page/single.html", "theme/_default/single.html") } - checkPageTitle(t, p, "Simple") - checkPageContent(t, p, "Summary Same Line
\n\nSome more text
\n") - checkPageSummary(t, p, "Summary Same Line
\n") - checkPageType(t, p, "page") - checkPageLayout(t, p, "page/single.html", "_default/single.html", "theme/page/single.html", "theme/_default/single.html") + + testAllMarkdownEnginesForPage(t, assertFunc, "simple", simplePageWithSummaryDelimiterSameLine) } func TestPageWithDate(t *testing.T) { - p, _ := NewPage("simple.md") - _, err := p.ReadFrom(strings.NewReader(simplePageRFC3339Date)) - p.Convert() - if err != nil { - t.Fatalf("Unable to create a page with frontmatter and body content: %s", err) - } - d, err := time.Parse(time.RFC3339, "2013-05-17T16:59:30Z") - if err != nil { - t.Fatalf("Unable to prase page.") + s := newSiteFromSources("simple.md", simplePageRFC3339Date) + + if err := buildSiteSkipRender(s); err != nil { + t.Fatalf("Failed to build site: %s", err) } + + require.Len(t, s.Pages, 1) + + p := s.Pages[0] + d, _ := time.Parse(time.RFC3339, "2013-05-17T16:59:30Z") + checkPageDate(t, p, d) } func TestWordCountWithAllCJKRunesWithoutHasCJKLanguage(t *testing.T) { testCommonResetState() - p, _ := NewPage("simple.md") - _, err := p.ReadFrom(strings.NewReader(simplePageWithAllCJKRunes)) - p.Convert() - p.analyzePage() - if err != nil { - t.Fatalf("Unable to create a page with frontmatter and body content: %s", err) + assertFunc := func(t *testing.T, ext string, p *Page) { + if p.WordCount != 8 { + t.Fatalf("[%s] incorrect word count for content '%s'. expected %v, got %v", ext, p.plain, 8, p.WordCount) + } } - if p.WordCount != 8 { - t.Fatalf("incorrect word count for content '%s'. expected %v, got %v", p.plain, 8, p.WordCount) - } + testAllMarkdownEnginesForPage(t, assertFunc, "simple", simplePageWithAllCJKRunes) } func TestWordCountWithAllCJKRunesHasCJKLanguage(t *testing.T) { testCommonResetState() - viper.Set("HasCJKLanguage", true) - p, _ := NewPage("simple.md") - _, err := p.ReadFrom(strings.NewReader(simplePageWithAllCJKRunes)) - p.Convert() - p.analyzePage() - if err != nil { - t.Fatalf("Unable to create a page with frontmatter and body content: %s", err) + assertFunc := func(t *testing.T, ext string, p *Page) { + if p.WordCount != 15 { + t.Fatalf("[%s] incorrect word count for content '%s'. expected %v, got %v", ext, p.plain, 15, p.WordCount) + } } - if p.WordCount != 15 { - t.Fatalf("incorrect word count for content '%s'. expected %v, got %v", p.plain, 15, p.WordCount) - } + testAllMarkdownEnginesForPage(t, assertFunc, "simple", simplePageWithAllCJKRunes) } func TestWordCountWithMainEnglishWithCJKRunes(t *testing.T) { @@ -682,69 +842,60 @@ func TestWordCountWithMainEnglishWithCJKRunes(t *testing.T) { viper.Set("HasCJKLanguage", true) - p, _ := NewPage("simple.md") - _, err := p.ReadFrom(strings.NewReader(simplePageWithMainEnglishWithCJKRunes)) - p.Convert() - p.analyzePage() - if err != nil { - t.Fatalf("Unable to create a page with frontmatter and body content: %s", err) - } + assertFunc := func(t *testing.T, ext string, p *Page) { + if p.WordCount != 74 { + t.Fatalf("[%s] incorrect word count for content '%s'. expected %v, got %v", ext, p.plain, 74, p.WordCount) + } - if p.WordCount != 74 { - t.Fatalf("incorrect word count for content '%s'. expected %v, got %v", p.plain, 74, p.WordCount) - } + if p.Summary != simplePageWithMainEnglishWithCJKRunesSummary { + t.Fatalf("[%s] incorrect Summary for content '%s'. expected %v, got %v", ext, p.plain, + simplePageWithMainEnglishWithCJKRunesSummary, p.Summary) + } - if p.Summary != simplePageWithMainEnglishWithCJKRunesSummary { - t.Fatalf("incorrect Summary for content '%s'. expected %v, got %v", p.plain, - simplePageWithMainEnglishWithCJKRunesSummary, p.Summary) } + + testAllMarkdownEnginesForPage(t, assertFunc, "simple", simplePageWithMainEnglishWithCJKRunes) } func TestWordCountWithIsCJKLanguageFalse(t *testing.T) { testCommonResetState() - viper.Set("HasCJKLanguage", true) - p, _ := NewPage("simple.md") - _, err := p.ReadFrom(strings.NewReader(simplePageWithIsCJKLanguageFalse)) - p.Convert() - p.analyzePage() - if err != nil { - t.Fatalf("Unable to create a page with frontmatter and body content: %s", err) - } + assertFunc := func(t *testing.T, ext string, p *Page) { + if p.WordCount != 75 { + t.Fatalf("[%s] incorrect word count for content '%s'. expected %v, got %v", ext, p.plain, 74, p.WordCount) + } - if p.WordCount != 75 { - t.Fatalf("incorrect word count for content '%s'. expected %v, got %v", p.plain, 75, p.WordCount) - } + if p.Summary != simplePageWithIsCJKLanguageFalseSummary { + t.Fatalf("[%s] incorrect Summary for content '%s'. expected %v, got %v", ext, p.plain, + simplePageWithIsCJKLanguageFalseSummary, p.Summary) + } - if p.Summary != simplePageWithIsCJKLanguageFalseSummary { - t.Fatalf("incorrect Summary for content '%s'. expected %v, got %v", p.plain, - simplePageWithIsCJKLanguageFalseSummary, p.Summary) } + + testAllMarkdownEnginesForPage(t, assertFunc, "simple", simplePageWithIsCJKLanguageFalse) + } func TestWordCount(t *testing.T) { - p, _ := NewPage("simple.md") - _, err := p.ReadFrom(strings.NewReader(simplePageWithLongContent)) - p.Convert() - p.analyzePage() - if err != nil { - t.Fatalf("Unable to create a page with frontmatter and body content: %s", err) - } - if p.WordCount != 483 { - t.Fatalf("incorrect word count. expected %v, got %v", 483, p.WordCount) - } + assertFunc := func(t *testing.T, ext string, p *Page) { + if p.WordCount != 483 { + t.Fatalf("[%s] incorrect word count. expected %v, got %v", ext, 483, p.WordCount) + } - if p.FuzzyWordCount != 500 { - t.Fatalf("incorrect word count. expected %v, got %v", 500, p.WordCount) - } + if p.FuzzyWordCount != 500 { + t.Fatalf("[%s] incorrect word count. expected %v, got %v", ext, 500, p.WordCount) + } + + if p.ReadingTime != 3 { + t.Fatalf("[%s] incorrect min read. expected %v, got %v", ext, 3, p.ReadingTime) + } - if p.ReadingTime != 3 { - t.Fatalf("incorrect min read. expected %v, got %v", 3, p.ReadingTime) + checkTruncation(t, p, true, "long page") } - checkTruncation(t, p, true, "long page") + testAllMarkdownEnginesForPage(t, assertFunc, "simple", simplePageWithLongContent) } func TestCreatePage(t *testing.T) { @@ -1049,15 +1200,18 @@ func TestPageSimpleMethods(t *testing.T) { } func TestChompBOM(t *testing.T) { - p, _ := NewPage("simple.md") const utf8BOM = "\xef\xbb\xbf" - _, err := p.ReadFrom(strings.NewReader(utf8BOM + simplePage)) - p.Convert() - if err != nil { - t.Fatalf("Unable to create a page with BOM prefixed frontmatter and body content: %s", err) + s := newSiteFromSources("simple.md", utf8BOM+simplePage) + + if err := buildSiteSkipRender(s); err != nil { + t.Fatalf("Failed to build site: %s", err) } + require.Len(t, s.Pages, 1) + + p := s.Pages[0] + checkPageTitle(t, p, "Simple") } diff --git a/hugolib/shortcode.go b/hugolib/shortcode.go index 2de00fa9079..b63ba4a4996 100644 --- a/hugolib/shortcode.go +++ b/hugolib/shortcode.go @@ -160,7 +160,11 @@ func HandleShortcodes(stringToParse string, page *Page, t tpl.Template) (string, } if len(tmpShortcodes) > 0 { - tmpContentWithTokensReplaced, err := replaceShortcodeTokens([]byte(tmpContent), shortcodePlaceholderPrefix, tmpShortcodes) + shortcodes, err := executeShortcodeFuncMap(tmpShortcodes) + if err != nil { + return "", err + } + tmpContentWithTokensReplaced, err := replaceShortcodeTokens([]byte(tmpContent), shortcodePlaceholderPrefix, shortcodes) if err != nil { return "", fmt.Errorf("Fail to replace short code tokens in %s:\n%s", page.BaseFileName(), err.Error()) @@ -274,7 +278,7 @@ func renderShortcode(sc shortcode, parent *ShortcodeWithPage, p *Page, t tpl.Tem return renderShortcodeWithPage(tmpl, data) } -func extractAndRenderShortcodes(stringToParse string, p *Page, t tpl.Template) (string, map[string]string, error) { +func extractAndRenderShortcodes(stringToParse string, p *Page, t tpl.Template) (string, map[string]func() (string, error), error) { if p.rendered { panic("Illegal state: Page already marked as rendered, please reuse the shortcodes") @@ -297,15 +301,32 @@ func extractAndRenderShortcodes(stringToParse string, p *Page, t tpl.Template) ( } -func renderShortcodes(shortcodes map[string]shortcode, p *Page, t tpl.Template) map[string]string { - renderedShortcodes := make(map[string]string) +var emptyShortcodeFn = func() (string, error) { return "", nil } + +func executeShortcodeFuncMap(funcs map[string]func() (string, error)) (map[string]string, error) { + result := make(map[string]string) + + for k, v := range funcs { + s, err := v() + if err != nil { + return nil, fmt.Errorf("Failed to execute shortcode with key %s: %s", k, err) + } + result[k] = s + } + + return result, nil +} + +func renderShortcodes(shortcodes map[string]shortcode, p *Page, t tpl.Template) map[string]func() (string, error) { + renderedShortcodes := make(map[string]func() (string, error)) for key, sc := range shortcodes { if sc.err != nil { // need to have something to replace with - renderedShortcodes[key] = "" + renderedShortcodes[key] = emptyShortcodeFn } else { - renderedShortcodes[key] = renderShortcode(sc, nil, p, t) + shorctode := sc + renderedShortcodes[key] = func() (string, error) { return renderShortcode(shorctode, nil, p, t), nil } } } diff --git a/hugolib/shortcode_test.go b/hugolib/shortcode_test.go index 5069fa195c4..dd16bff07d9 100644 --- a/hugolib/shortcode_test.go +++ b/hugolib/shortcode_test.go @@ -484,9 +484,34 @@ e`, {"sect/doc8.rst", `**Shortcodes:** *b: {{< b >}} c: {{% c %}}*`, filepath.FromSlash("sect/doc8/index.html"), "Shortcodes: b: b c: c
\nShortcodes: b: b c: c
\n"}, + // Issue #1229: Menus not available in shortcode. + {"sect/doc10.md", `--- +menu: + main: + identifier: 'parent' +tags: +- Menu +--- +**Menus:** {{< menu >}}`, + filepath.FromSlash("sect/doc10/index.html"), + "Menus: 1
\n"}, + // Issue #2323: Taxonomies not available in shortcode. + {"sect/doc11.md", `--- +tags: +- Bugs +--- +**Tags:** {{< tags >}}`, + filepath.FromSlash("sect/doc11/index.html"), + "Tags: 2
\n"}, } sources := make([]source.ByteSource, len(tests)) @@ -507,6 +532,8 @@ e`, templ.AddInternalShortcode("b.html", `b`) templ.AddInternalShortcode("c.html", `c`) templ.AddInternalShortcode("d.html", `d`) + templ.AddInternalShortcode("menu.html", `{{ len (index .Page.Menus "main").Children }}`) + templ.AddInternalShortcode("tags.html", `{{ len .Page.Site.Taxonomies.tags }}`) return nil @@ -540,7 +567,7 @@ e`, content := helpers.ReaderToString(file) if content != test.expected { - t.Errorf("%s content expected:\n%q\ngot:\n%q", test.outFile, test.expected, content) + t.Fatalf("%s content expected:\n%q\ngot:\n%q", test.outFile, test.expected, content) } } diff --git a/hugolib/site.go b/hugolib/site.go index 59b8379dc5d..03e697130bd 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -113,6 +113,26 @@ func newSiteDefaultLang() *Site { return NewSite(newDefaultLanguage()) } +// Convenience func used in tests. +func newSiteFromSources(pathContentPairs ...string) *Site { + if len(pathContentPairs)%2 != 0 { + panic("pathContentPairs must come in pairs") + } + + sources := make([]source.ByteSource, 0) + + for i := 0; i < len(pathContentPairs); i += 2 { + path := pathContentPairs[i] + content := pathContentPairs[i+1] + sources = append(sources, source.ByteSource{Name: filepath.FromSlash(path), Content: []byte(content)}) + } + + return &Site{ + Source: &source.InMemorySource{ByteSource: sources}, + Language: newDefaultLanguage(), + } +} + type targetList struct { page target.Output pageUgly target.Output diff --git a/hugolib/site_test.go b/hugolib/site_test.go index b9e2d346b59..f98bc6a7dec 100644 --- a/hugolib/site_test.go +++ b/hugolib/site_test.go @@ -30,6 +30,7 @@ import ( "github.com/spf13/hugo/target" "github.com/spf13/viper" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) const ( @@ -84,10 +85,16 @@ func pageMust(p *Page, err error) *Page { } func TestDegenerateRenderThingMissingTemplate(t *testing.T) { - p, _ := NewPageFrom(strings.NewReader(pageSimpleTitle), "content/a/file.md") - p.Convert() - s := new(Site) - s.prepTemplates(nil) + s := newSiteFromSources("content/a/file.md", pageSimpleTitle) + + if err := buildSiteSkipRender(s); err != nil { + t.Fatalf("Failed to build site: %s", err) + } + + require.Len(t, s.Pages, 1) + + p := s.Pages[0] + err := s.renderThing(p, "foobar", nil) if err == nil { t.Errorf("Expected err to be returned when missing the template.")