diff --git a/helpers/content.go b/helpers/content.go index e955b0004c9..b9281acf406 100644 --- a/helpers/content.go +++ b/helpers/content.go @@ -41,6 +41,8 @@ var SummaryLength = 70 // SummaryDivider denotes where content summarization should end. The default is "". var SummaryDivider = []byte("") +var summaryDividerAndNewLines = []byte("\n\n") + // Blackfriday holds configuration values for Blackfriday rendering. type Blackfriday struct { Smartypants bool @@ -390,8 +392,21 @@ func WordCount(s string) map[string]int { } // RemoveSummaryDivider removes summary-divider from content. +// TODO(bep) ml remove func RemoveSummaryDivider(content []byte) []byte { - return bytes.Replace(content, SummaryDivider, []byte(""), -1) + b := bytes.Replace(content, summaryDividerAndNewLines, []byte(""), 1) + if len(b) != len(content) { + return b + } + return bytes.Replace(content, SummaryDivider, []byte(""), 1) +} + +func removeInternalSummaryDivider(content []byte) []byte { + b := bytes.Replace(content, summaryDividerAndNewLines, []byte(""), 1) + if len(b) != len(content) { + return b + } + return bytes.Replace(content, SummaryDivider, []byte(""), 1) } // TruncateWordsByRune truncates words by runes. diff --git a/hugolib/handler_meta.go b/hugolib/handler_meta.go index 0825ef1e68f..36b0f5a2a10 100644 --- a/hugolib/handler_meta.go +++ b/hugolib/handler_meta.go @@ -75,8 +75,6 @@ func (mh *MetaHandle) Convert(i interface{}, s *Site, results HandleResults) { } results <- h.PageConvert(p, s.Tmpl) - p.setSummary() - p.analyzePage() } } diff --git a/hugolib/handler_page.go b/hugolib/handler_page.go index 73a972446ce..04af20ceb76 100644 --- a/hugolib/handler_page.go +++ b/hugolib/handler_page.go @@ -14,10 +14,11 @@ package hugolib import ( + "bytes" + "github.com/spf13/hugo/helpers" "github.com/spf13/hugo/source" "github.com/spf13/hugo/tpl" - jww "github.com/spf13/jwalterweatherman" "github.com/spf13/viper" ) @@ -67,18 +68,7 @@ type htmlHandler struct { func (h htmlHandler) Extensions() []string { return []string{"html", "htm"} } func (h htmlHandler) PageConvert(p *Page, t tpl.Template) HandledResult { p.ProcessShortcodes(t) - var err error - - if len(p.contentShortCodes) > 0 { - p.rawContent, err = replaceShortcodeTokens(p.rawContent, shortcodePlaceholderPrefix, p.contentShortCodes) - if err != nil { - jww.FATAL.Printf("Failed to replace short code tokens in %s:\n%s", p.BaseFileName(), err.Error()) - return HandledResult{err: err} - } - } - - p.Content = helpers.BytesToHTML(p.rawContent) return HandledResult{err: nil} } @@ -112,27 +102,22 @@ func (h mmarkHandler) PageConvert(p *Page, t tpl.Template) HandledResult { func commonConvert(p *Page, t tpl.Template) HandledResult { p.ProcessShortcodes(t) - var err error - + // TODO(bep) these page handlers need to be re-evaluated, as it is hard to + // process a page in isolation. See the new preRender func. + // TODO(bep) ml not so raw anymore, but do we need to keep it raw? if viper.GetBool("EnableEmoji") { p.rawContent = helpers.Emojify(p.rawContent) } - renderedContent := p.renderContent(helpers.RemoveSummaryDivider(p.rawContent)) - - if len(p.contentShortCodes) > 0 { - renderedContent, err = replaceShortcodeTokens(renderedContent, shortcodePlaceholderPrefix, p.contentShortCodes) - - if err != nil { - jww.FATAL.Printf("Failed to replace shortcode tokens in %s:\n%s", p.BaseFileName(), err.Error()) - return HandledResult{err: err} - } - } - - tmpContent, tmpTableOfContents := helpers.ExtractTOC(renderedContent) + // TODO(bep) ml we let the summary divider survive the rendering. Must check if + // it actually survives, replace it with something more robus, or maybe + // rethink this fragile concept. + //p.rawContent = p.renderContent(helpers.RemoveSummaryDivider(p.rawContent)) + // We have to replace the with something that survives all the + // rendering engines. + // TODO(bep) inline replace + p.rawContent = bytes.Replace(p.rawContent, []byte(helpers.SummaryDivider), internalSummaryDivider, 1) + p.rawContent = p.renderContent(p.rawContent) - p.Content = helpers.BytesToHTML(tmpContent) - p.TableOfContents = helpers.BytesToHTML(tmpTableOfContents) - p.rendered = true return HandledResult{err: nil} } diff --git a/hugolib/hugo_sites.go b/hugolib/hugo_sites.go index 2dd1bb9bef8..76f9979fd90 100644 --- a/hugolib/hugo_sites.go +++ b/hugolib/hugo_sites.go @@ -16,8 +16,11 @@ package hugolib import ( "errors" "strings" + "sync" "time" + "github.com/spf13/hugo/helpers" + "github.com/spf13/viper" "github.com/fsnotify/fsnotify" @@ -106,24 +109,27 @@ func (h HugoSites) Build(config BuildCfg) error { } for _, s := range h.Sites { - if err := s.PostProcess(); err != nil { return err } + } + + if err := h.preRender(); err != nil { + return err + } + + for _, s := range h.Sites { if !config.skipRender { if err := s.Render(); err != nil { return err } + if config.PrintStats { + s.Stats() + } } - - if config.PrintStats { - s.Stats() - } - // TODO(bep) ml lang in site.Info? - // TODO(bep) ml Page sorting? } if config.PrintStats { @@ -153,22 +159,26 @@ func (h HugoSites) Rebuild(config BuildCfg, events ...fsnotify.Event) error { // Assign pages to sites per translation. h.setupTranslations(firstSite) - for _, s := range h.Sites { - - if sourceChanged { + if sourceChanged { + for _, s := range h.Sites { if err := s.PostProcess(); err != nil { return err } } + } - if !config.skipRender { + if err := h.preRender(); err != nil { + return err + } + + if !config.skipRender { + for _, s := range h.Sites { if err := s.Render(); err != nil { return err } - } - - if config.PrintStats { - s.Stats() + if config.PrintStats { + s.Stats() + } } } @@ -219,6 +229,87 @@ func (s *HugoSites) setupTranslations(master *Site) { } } +// preRender performs build tasks that needs to be done as late as possible. +// Shortcode handling is the main task in here. +// TODO(bep) We need to look at the whole handler-chain construct witht he below in mind. +func (h *HugoSites) preRender() error { + pageChan := make(chan *Page) + + wg := &sync.WaitGroup{} + + // We want all the pages, so just pick one. + s := h.Sites[0] + + for i := 0; i < getGoMaxProcs()*4; i++ { + wg.Add(1) + go func(pages <-chan *Page, wg *sync.WaitGroup) { + defer wg.Done() + for p := range pages { + if err := handleShortcodes(p, s.Tmpl); err != nil { + jww.ERROR.Printf("Failed to handle shortcodes for page %s: %s", p.BaseFileName(), err) + } + + if p.Markup == "markdown" { + tmpContent, tmpTableOfContents := helpers.ExtractTOC(p.rawContent) + p.TableOfContents = helpers.BytesToHTML(tmpTableOfContents) + p.rawContent = tmpContent + } + + if p.Markup != "html" { + + // Now we know enough to create a summary of the page and count some words + summaryContent, err := p.setUserDefinedSummaryIfProvided() + + if err != nil { + jww.ERROR.Printf("Failed to set use defined summary: %s", err) + } else if summaryContent != nil { + p.rawContent = summaryContent.content + } + + p.Content = helpers.BytesToHTML(p.rawContent) + p.rendered = true + + if summaryContent == nil { + p.setAutoSummary() + } + } + + //analyze for raw stats + p.analyzePage() + } + }(pageChan, wg) + } + + for _, p := range s.AllPages { + pageChan <- p + } + + close(pageChan) + + wg.Wait() + + return nil +} + +func handleShortcodes(p *Page, t tpl.Template) error { + if len(p.contentShortCodes) > 0 { + jww.DEBUG.Printf("Replace %d shortcodes in %q", len(p.contentShortCodes), p.BaseFileName()) + shortcodes, err := executeShortcodeFuncMap(p.contentShortCodes) + + if err != nil { + return err + } + + p.rawContent, err = replaceShortcodeTokens(p.rawContent, shortcodePlaceholderPrefix, shortcodes) + + if err != nil { + jww.FATAL.Printf("Failed to replace short code tokens in %s:\n%s", p.BaseFileName(), err.Error()) + } + } + + return nil +} + func (s *Site) updateBuildStats(page *Page) { if page.IsDraft() { s.draftCount++ diff --git a/hugolib/hugo_sites_test.go b/hugolib/hugo_sites_test.go index fc48011159b..b63792a3bc7 100644 --- a/hugolib/hugo_sites_test.go +++ b/hugolib/hugo_sites_test.go @@ -37,6 +37,11 @@ func testCommonResetState() { viper.Set("PublishDir", "public") viper.Set("RSSUri", "rss") + viper.Set("Taxonomies", map[string]interface{}{ + "tag": "tags", + "category": "categories", + }) + if err := hugofs.Source().Mkdir("content", 0755); err != nil { panic("Content folder creation failed.") } diff --git a/hugolib/page.go b/hugolib/page.go index 4248ff893ae..493e5b51228 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -70,7 +70,7 @@ type Page struct { linkTitle string frontmatter []byte rawContent []byte - contentShortCodes map[string]string // TODO(bep) this shouldn't be needed. + contentShortCodes map[string]func() (string, error) shortcodes map[string]shortcode plain string // TODO should be []byte plainWords []string @@ -212,46 +212,111 @@ func (p *Page) lineNumRawContentStart() int { return bytes.Count(p.frontmatter, []byte("\n")) + 1 } -func (p *Page) setSummary() { +var ( + internalSummaryDivider = []byte("HUGOMORE42") + internalSummaryDividerAndNewLines = []byte("HUGOMORE42\n\n") +) - // at this point, p.rawContent contains placeholders for the short codes, - // rendered and ready in p.contentShortcodes +// Returns the page as summary and main if a user defined split is provided. +func (p *Page) setUserDefinedSummaryIfProvided() (*summaryContent, error) { - if bytes.Contains(p.rawContent, helpers.SummaryDivider) { - sections := bytes.Split(p.rawContent, helpers.SummaryDivider) - header := sections[0] - p.Truncated = true - if len(sections[1]) < 20 { - // only whitespace? - p.Truncated = len(bytes.Trim(sections[1], " \n\r")) > 0 - } + sc := splitUserDefinedSummaryAndContent(p.Markup, p.rawContent) - // TODO(bep) consider doing this once only - renderedHeader := p.renderBytes(header) - if len(p.contentShortCodes) > 0 { - tmpContentWithTokensReplaced, err := - replaceShortcodeTokens(renderedHeader, shortcodePlaceholderPrefix, p.contentShortCodes) - if err != nil { - jww.FATAL.Printf("Failed to replace short code tokens in Summary for %s:\n%s", p.BaseFileName(), err.Error()) - } else { - renderedHeader = tmpContentWithTokensReplaced - } - } - p.Summary = helpers.BytesToHTML(renderedHeader) - } else { - // If hugo defines split: - // render, strip html, then split - var summary string - var truncated bool - if p.isCJKLanguage { - summary, truncated = helpers.TruncateWordsByRune(p.PlainWords(), helpers.SummaryLength) - } else { - summary, truncated = helpers.TruncateWordsToWholeSentence(p.PlainWords(), helpers.SummaryLength) - } - p.Summary = template.HTML(summary) - p.Truncated = truncated + if sc == nil { + // No divider found + return nil, nil + } + p.Truncated = true + if len(sc.content) < 20 { + // only whitespace? + p.Truncated = len(bytes.Trim(sc.content, " \n\r")) > 0 } + + p.Summary = helpers.BytesToHTML(sc.summary) + + return sc, nil +} + +// Make this explicit so there is no doubt about what is what. +type summaryContent struct { + summary []byte + content []byte + contentWithoutSummary []byte +} + +func splitUserDefinedSummaryAndContent(markup string, c []byte) *summaryContent { + startDivider := bytes.Index(c, internalSummaryDivider) + + if startDivider == -1 { + return nil + } + + endDivider := startDivider + len(internalSummaryDivider) + endSummary := startDivider + + var ( + startMarkup []byte + endMarkup []byte + addDiv bool + divStart = []byte("
") + ) + + switch markup { + default: + startMarkup = []byte("

") + endMarkup = []byte("

") + case "asciidoc": + startMarkup = []byte("
") + endMarkup = []byte("
") + case "rst": + startMarkup = []byte("

") + endMarkup = []byte("

") + addDiv = true + } + + // Find the closest end/start markup string to the divider + //firstStart := bytes.Index(c[:startDivider], startMarkup) + fromIdx := bytes.LastIndex(c[:startDivider], startMarkup) + fromStart := startDivider - fromIdx - len(startMarkup) + fromEnd := bytes.Index(c[endDivider:], endMarkup) + + if fromEnd != -1 && fromEnd <= fromStart { + endSummary = startDivider + fromEnd + len(endMarkup) + } else if fromStart != -1 { + endSummary = startDivider - fromStart - len(startMarkup) + } + + withoutDivider := bytes.TrimSpace(append(c[:startDivider], c[endDivider:]...)) + contentWithoutSummary := bytes.TrimSpace(withoutDivider[endSummary:]) + summary := bytes.TrimSpace(withoutDivider[:endSummary]) + + if addDiv { + // For the rst + summary = append(append([]byte(nil), summary...), []byte("
")...) + // TODO(bep) include the document class, maybe + contentWithoutSummary = append(divStart, contentWithoutSummary...) + } + + return &summaryContent{ + summary: summary, + content: withoutDivider, + contentWithoutSummary: contentWithoutSummary, + } +} + +func (p *Page) setAutoSummary() error { + var summary string + var truncated bool + if p.isCJKLanguage { + summary, truncated = helpers.TruncateWordsByRune(p.PlainWords(), helpers.SummaryLength) + } else { + summary, truncated = helpers.TruncateWordsToWholeSentence(p.PlainWords(), helpers.SummaryLength) + } + p.Summary = template.HTML(summary) + p.Truncated = truncated + + return nil } func (p *Page) renderBytes(content []byte) []byte { @@ -972,27 +1037,6 @@ func (p *Page) ProcessShortcodes(t tpl.Template) { } -// TODO(spf13): Remove this entirely -// Here for backwards compatibility & testing. Only works in isolation -func (p *Page) Convert() error { - var h Handler - if p.Markup != "" { - h = FindHandler(p.Markup) - } else { - h = FindHandler(p.File.Extension()) - } - if h != nil { - h.PageConvert(p, tpl.T()) - } - - //// now we know enough to create a summary of the page and count some words - p.setSummary() - //analyze for raw stats - p.analyzePage() - - return nil -} - func (p *Page) FullFilePath() string { return filepath.Join(p.Dir(), p.LogicalName()) } diff --git a/hugolib/page_test.go b/hugolib/page_test.go index 6fd797830d0..6927e6e8219 100644 --- a/hugolib/page_test.go +++ b/hugolib/page_test.go @@ -27,6 +27,7 @@ import ( "github.com/spf13/hugo/helpers" "github.com/spf13/viper" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) var emptyPage = "" @@ -130,6 +131,15 @@ Summary Next Line Some more text +` + + simplePageWithSummaryDelimiterAndMarkdownThatCrossesBorder = `--- +title: Simple +--- +The [best static site generator][hugo].[^1] + +[hugo]: http://gohugo.io/ +[^1]: Many people say so. ` simplePageWithShortcodeInSummary = `--- title: Simple @@ -485,21 +495,33 @@ func checkPageTitle(t *testing.T, page *Page, title string) { } } -func checkPageContent(t *testing.T, page *Page, content string) { - if page.Content != template.HTML(content) { - t.Fatalf("Page content is:\n%q\nExpected:\n%q", page.Content, content) +func checkPageContent(t *testing.T, page *Page, content string, msg ...interface{}) { + a := normalizeContent(content) + b := normalizeContent(string(page.Content)) + if a != b { + t.Fatalf("Page content is:\n%q\nExpected:\n%q (%q)", b, a, msg) } } +func normalizeContent(c string) string { + norm := strings.Replace(c, "\n", "", -1) + norm = strings.Replace(norm, " ", " ", -1) + norm = strings.Replace(norm, " ", " ", -1) + norm = strings.Replace(norm, " ", " ", -1) + return strings.TrimSpace(norm) +} + func checkPageTOC(t *testing.T, page *Page, toc string) { if page.TableOfContents != template.HTML(toc) { t.Fatalf("Page TableOfContents is: %q.\nExpected %q", page.TableOfContents, toc) } } -func checkPageSummary(t *testing.T, page *Page, summary string) { - if page.Summary != template.HTML(summary) { - t.Fatalf("Page summary is: %q.\nExpected %q", page.Summary, summary) +func checkPageSummary(t *testing.T, page *Page, summary string, msg ...interface{}) { + a := normalizeContent(string(page.Summary)) + b := normalizeContent(summary) + if a != b { + t.Fatalf("Page summary is:\n%q.\nExpected\n%q (%q)", a, b, msg) } } @@ -534,147 +556,285 @@ func checkTruncation(t *testing.T, page *Page, shouldBe bool, msg string) { } } +func normalizeExpected(ext, str string) string { + str = normalizeContent(str) + switch ext { + default: + return str + case "html": + return strings.Trim(helpers.StripHTML(str), " ") + case "ad": + paragraphs := strings.Split(str, "

") + expected := "" + for _, para := range paragraphs { + if para == "" { + continue + } + expected += fmt.Sprintf("
\n%s

\n", para) + } + return expected + case "rst": + return fmt.Sprintf("
\n\n\n%s
", str) + } +} + +func testAllMarkdownEnginesForPage(t *testing.T, + assertFunc func(t *testing.T, ext string, p *Page), baseFilename, pageContent string) { + + engines := []struct { + ext string + shouldExecute func() bool + }{ + {"ad", func() bool { return helpers.HasAsciidoc() }}, + {"md", func() bool { return true }}, + {"mmark", func() bool { return true }}, + // TODO(bep) figure a way to include this without too much work.{"html", func() bool { return true }}, + {"rst", func() bool { return helpers.HasRst() }}, + } + + for _, e := range engines { + if !e.shouldExecute() { + continue + } + + filename := baseFilename + "." + e.ext + + s := newSiteFromSources(filename, pageContent) + + if err := buildSiteSkipRender(s); err != nil { + t.Fatalf("Failed to build site: %s", err) + } + + require.Len(t, s.Pages, 1) + + p := s.Pages[0] + + assertFunc(t, e.ext, p) + + } + +} + func TestCreateNewPage(t *testing.T) { - p, _ := NewPage("simple.md") - _, err := p.ReadFrom(strings.NewReader(simplePage)) - p.Convert() - if err != nil { - t.Fatalf("Unable to create a page with frontmatter and body content: %s", err) + assertFunc := func(t *testing.T, ext string, p *Page) { + assert.False(t, p.IsHome) + checkPageTitle(t, p, "Simple") + checkPageContent(t, p, normalizeExpected(ext, "

Simple Page

\n")) + checkPageSummary(t, p, "Simple Page") + checkPageType(t, p, "page") + checkPageLayout(t, p, "page/single.html", "_default/single.html", "theme/page/single.html", "theme/_default/single.html") + checkTruncation(t, p, false, "simple short page") } - assert.False(t, p.IsHome) - checkPageTitle(t, p, "Simple") - checkPageContent(t, p, "

Simple Page

\n") - checkPageSummary(t, p, "Simple Page") - checkPageType(t, p, "page") - checkPageLayout(t, p, "page/single.html", "_default/single.html", "theme/page/single.html", "theme/_default/single.html") - checkTruncation(t, p, false, "simple short page") + testAllMarkdownEnginesForPage(t, assertFunc, "simple", simplePage) +} + +func TestSplitSummaryAndContent(t *testing.T) { + + for i, this := range []struct { + markup string + content string + expectedSummary string + expectedContent string + expectedContentWithoutSummary string + }{ + {"markdown", `

Summary Same LineHUGOMORE42

+ +

Some more text

`, "

Summary Same Line

", "

Summary Same Line

\n\n

Some more text

", "

Some more text

"}, + {"asciidoc", `

sn

HUGOMORE42Some more text

`, + "

sn

", + "

sn

Some more text

", + "

Some more text

"}, + {"rst", + "

Summary Next Line

HUGOMORE42Some more text

", + "

Summary Next Line

", + "

Summary Next Line

Some more text

", + "

Some more text

"}, + {"markdown", "

a

b

HUGOMORE42c

", "

a

b

", "

a

b

c

", "

c

"}, + {"markdown", "

a

b

cHUGOMORE42

", "

a

b

c

", "

a

b

c

", ""}, + {"markdown", "

a

bHUGOMORE42

c

", "

a

b

", "

a

b

c

", "

c

"}, + {"markdown", "

aHUGOMORE42

b

c

", "

a

", "

a

b

c

", "

b

c

"}, + } { + + sc := splitUserDefinedSummaryAndContent(this.markup, []byte(this.content)) + + require.NotNil(t, sc, fmt.Sprintf("[%d] Nil %s", i, this.markup)) + require.Equal(t, this.expectedSummary, string(sc.summary), fmt.Sprintf("[%d] Summary markup %s", i, this.markup)) + require.Equal(t, this.expectedContent, string(sc.content), fmt.Sprintf("[%d] Content markup %s", i, this.markup)) + require.Equal(t, this.expectedContentWithoutSummary, string(sc.contentWithoutSummary), fmt.Sprintf("[%d] Content without summary, markup %s", i, this.markup)) + } + + if true { + return + } + + ad := `

sn

+
+

HUGOMORE42 +Some more text

+
+` + + md := `

Summary Same LineHUGOMORE42

+ +

Some more text

` + + sc := splitUserDefinedSummaryAndContent("markdown", []byte(md)) + + require.Equal(t, "adf", string(sc.summary)) + require.Equal(t, "asdf", string(sc.content)) + + if true { + return + } + sc = splitUserDefinedSummaryAndContent("asciidoc", []byte(ad)) + require.Equal(t, "

sn

", string(sc.summary)) + require.Equal(t, "\n
\n

\nSome more text

\n
\n", string(sc.summary)) + } func TestPageWithDelimiter(t *testing.T) { - p, _ := NewPage("simple.md") - _, err := p.ReadFrom(strings.NewReader(simplePageWithSummaryDelimiter)) - p.Convert() - if err != nil { - t.Fatalf("Unable to create a page with frontmatter and body content: %s", err) + + assertFunc := func(t *testing.T, ext string, p *Page) { + checkPageTitle(t, p, "Simple") + checkPageContent(t, p, normalizeExpected(ext, "

Summary Next Line

\n\n

Some more text

\n"), ext) + checkPageSummary(t, p, normalizeExpected(ext, "

Summary Next Line

"), ext) + checkPageType(t, p, "page") + checkPageLayout(t, p, "page/single.html", "_default/single.html", "theme/page/single.html", "theme/_default/single.html") + checkTruncation(t, p, true, "page with summary delimiter") + } + + testAllMarkdownEnginesForPage(t, assertFunc, "simple", simplePageWithSummaryDelimiter) +} + +// Issue #1076 +func TestPageWithDelimiterForMarkdownThatCrossesBorder(t *testing.T) { + s := newSiteFromSources("simple.md", simplePageWithSummaryDelimiterAndMarkdownThatCrossesBorder) + + if err := buildSiteSkipRender(s); err != nil { + t.Fatalf("Failed to build site: %s", err) + } + + require.Len(t, s.Pages, 1) + + p := s.Pages[0] + + if p.Summary != template.HTML("

The best static site generator.1\n

") { + t.Fatalf("Got summary:\n%q", p.Summary) + } + + if p.Content != template.HTML("

The best static site generator.1\n

\n
\n\n
\n\n
    \n
  1. Many people say so.\n [return]
  2. \n
\n
") { + t.Fatalf("Got content:\n%q", p.Content) } - checkPageTitle(t, p, "Simple") - checkPageContent(t, p, "

Summary Next Line

\n\n

Some more text

\n") - checkPageSummary(t, p, "

Summary Next Line

\n") - checkPageType(t, p, "page") - checkPageLayout(t, p, "page/single.html", "_default/single.html", "theme/page/single.html", "theme/_default/single.html") - checkTruncation(t, p, true, "page with summary delimiter") } func TestPageWithShortCodeInSummary(t *testing.T) { - s := new(Site) - s.prepTemplates(nil) - p, _ := NewPage("simple.md") - _, err := p.ReadFrom(strings.NewReader(simplePageWithShortcodeInSummary)) - if err != nil { - t.Fatalf("Unable to create a page with frontmatter and body content: %s", err) + + assertFunc := func(t *testing.T, ext string, p *Page) { + checkPageTitle(t, p, "Simple") + checkPageContent(t, p, normalizeExpected(ext, "

Summary Next Line.

.\nMore text here.

Some more text

"), ext) + checkPageSummary(t, p, "Summary Next Line. . More text here. Some more text", ext) + checkPageType(t, p, "page") + checkPageLayout(t, p, "page/single.html", "_default/single.html", "theme/page/single.html", "theme/_default/single.html") } - p.Convert() - checkPageTitle(t, p, "Simple") - checkPageContent(t, p, "

Summary Next Line. \n

\n \n \n \n \n
\n.\nMore text here.

\n\n

Some more text

\n") - checkPageSummary(t, p, "Summary Next Line. . More text here. Some more text") - checkPageType(t, p, "page") - checkPageLayout(t, p, "page/single.html", "_default/single.html", "theme/page/single.html", "theme/_default/single.html") + testAllMarkdownEnginesForPage(t, assertFunc, "simple", simplePageWithShortcodeInSummary) } func TestPageWithEmbeddedScriptTag(t *testing.T) { - p, _ := NewPage("simple.md") - _, err := p.ReadFrom(strings.NewReader(simplePageWithEmbeddedScript)) - p.Convert() - if err != nil { - t.Fatalf("Unable to create a page with frontmatter and body content: %s", err) + + assertFunc := func(t *testing.T, ext string, p *Page) { + if ext == "ad" || ext == "rst" { + // TOD(bep) + return + } + checkPageContent(t, p, "\n", ext) } - checkPageContent(t, p, "\n") + + testAllMarkdownEnginesForPage(t, assertFunc, "simple", simplePageWithEmbeddedScript) } func TestPageWithAdditionalExtension(t *testing.T) { - p, _ := NewPage("simple.md") - _, err := p.ReadFrom(strings.NewReader(simplePageWithAdditionalExtension)) - p.Convert() - if err != nil { - t.Fatalf("Unable to create a page with frontmatter and body content: %s", err) + s := newSiteFromSources("simple.md", simplePageWithAdditionalExtension) + + if err := buildSiteSkipRender(s); err != nil { + t.Fatalf("Failed to build site: %s", err) } + + require.Len(t, s.Pages, 1) + + p := s.Pages[0] + checkPageContent(t, p, "

first line.
\nsecond line.

\n\n

fourth line.

\n") } func TestTableOfContents(t *testing.T) { - p, _ := NewPage("tocpage.md") - _, err := p.ReadFrom(strings.NewReader(pageWithToC)) - p.Convert() - if err != nil { - t.Fatalf("Unable to create a page with frontmatter and body content: %s", err) + s := newSiteFromSources("tocpage.md", pageWithToC) + + if err := buildSiteSkipRender(s); err != nil { + t.Fatalf("Failed to build site: %s", err) } + + require.Len(t, s.Pages, 1) + + p := s.Pages[0] + checkPageContent(t, p, "\n\n

For some moments the old man did not reply. He stood with bowed head, buried in deep thought. But at last he spoke.

\n\n

AA

\n\n

I have no idea, of course, how long it took me to reach the limit of the plain,\nbut at last I entered the foothills, following a pretty little canyon upward\ntoward the mountains. Beside me frolicked a laughing brooklet, hurrying upon\nits noisy way down to the silent sea. In its quieter pools I discovered many\nsmall fish, of four-or five-pound weight I should imagine. In appearance,\nexcept as to size and color, they were not unlike the whale of our own seas. As\nI watched them playing about I discovered, not only that they suckled their\nyoung, but that at intervals they rose to the surface to breathe as well as to\nfeed upon certain grasses and a strange, scarlet lichen which grew upon the\nrocks just above the water line.

\n\n

AAA

\n\n

I remember I felt an extraordinary persuasion that I was being played with,\nthat presently, when I was upon the very verge of safety, this mysterious\ndeath–as swift as the passage of light–would leap after me from the pit about\nthe cylinder and strike me down. ## BB

\n\n

BBB

\n\n

“You’re a great Granser,” he cried delightedly, “always making believe them little marks mean something.”

\n") checkPageTOC(t, p, "") } func TestPageWithMoreTag(t *testing.T) { - p, _ := NewPage("simple.md") - _, err := p.ReadFrom(strings.NewReader(simplePageWithSummaryDelimiterSameLine)) - p.Convert() - if err != nil { - t.Fatalf("Unable to create a page with frontmatter and body content: %s", err) + + assertFunc := func(t *testing.T, ext string, p *Page) { + checkPageTitle(t, p, "Simple") + checkPageContent(t, p, normalizeExpected(ext, "

Summary Same Line

\n\n

Some more text

\n")) + checkPageSummary(t, p, normalizeExpected(ext, "

Summary Same Line

")) + checkPageType(t, p, "page") + checkPageLayout(t, p, "page/single.html", "_default/single.html", "theme/page/single.html", "theme/_default/single.html") } - checkPageTitle(t, p, "Simple") - checkPageContent(t, p, "

Summary Same Line

\n\n

Some more text

\n") - checkPageSummary(t, p, "

Summary Same Line

\n") - checkPageType(t, p, "page") - checkPageLayout(t, p, "page/single.html", "_default/single.html", "theme/page/single.html", "theme/_default/single.html") + + testAllMarkdownEnginesForPage(t, assertFunc, "simple", simplePageWithSummaryDelimiterSameLine) } func TestPageWithDate(t *testing.T) { - p, _ := NewPage("simple.md") - _, err := p.ReadFrom(strings.NewReader(simplePageRFC3339Date)) - p.Convert() - if err != nil { - t.Fatalf("Unable to create a page with frontmatter and body content: %s", err) - } - d, err := time.Parse(time.RFC3339, "2013-05-17T16:59:30Z") - if err != nil { - t.Fatalf("Unable to prase page.") + s := newSiteFromSources("simple.md", simplePageRFC3339Date) + + if err := buildSiteSkipRender(s); err != nil { + t.Fatalf("Failed to build site: %s", err) } + + require.Len(t, s.Pages, 1) + + p := s.Pages[0] + d, _ := time.Parse(time.RFC3339, "2013-05-17T16:59:30Z") + checkPageDate(t, p, d) } func TestWordCountWithAllCJKRunesWithoutHasCJKLanguage(t *testing.T) { testCommonResetState() - p, _ := NewPage("simple.md") - _, err := p.ReadFrom(strings.NewReader(simplePageWithAllCJKRunes)) - p.Convert() - p.analyzePage() - if err != nil { - t.Fatalf("Unable to create a page with frontmatter and body content: %s", err) + assertFunc := func(t *testing.T, ext string, p *Page) { + if p.WordCount != 8 { + t.Fatalf("[%s] incorrect word count for content '%s'. expected %v, got %v", ext, p.plain, 8, p.WordCount) + } } - if p.WordCount != 8 { - t.Fatalf("incorrect word count for content '%s'. expected %v, got %v", p.plain, 8, p.WordCount) - } + testAllMarkdownEnginesForPage(t, assertFunc, "simple", simplePageWithAllCJKRunes) } func TestWordCountWithAllCJKRunesHasCJKLanguage(t *testing.T) { testCommonResetState() - viper.Set("HasCJKLanguage", true) - p, _ := NewPage("simple.md") - _, err := p.ReadFrom(strings.NewReader(simplePageWithAllCJKRunes)) - p.Convert() - p.analyzePage() - if err != nil { - t.Fatalf("Unable to create a page with frontmatter and body content: %s", err) + assertFunc := func(t *testing.T, ext string, p *Page) { + if p.WordCount != 15 { + t.Fatalf("[%s] incorrect word count for content '%s'. expected %v, got %v", ext, p.plain, 15, p.WordCount) + } } - if p.WordCount != 15 { - t.Fatalf("incorrect word count for content '%s'. expected %v, got %v", p.plain, 15, p.WordCount) - } + testAllMarkdownEnginesForPage(t, assertFunc, "simple", simplePageWithAllCJKRunes) } func TestWordCountWithMainEnglishWithCJKRunes(t *testing.T) { @@ -682,69 +842,60 @@ func TestWordCountWithMainEnglishWithCJKRunes(t *testing.T) { viper.Set("HasCJKLanguage", true) - p, _ := NewPage("simple.md") - _, err := p.ReadFrom(strings.NewReader(simplePageWithMainEnglishWithCJKRunes)) - p.Convert() - p.analyzePage() - if err != nil { - t.Fatalf("Unable to create a page with frontmatter and body content: %s", err) - } + assertFunc := func(t *testing.T, ext string, p *Page) { + if p.WordCount != 74 { + t.Fatalf("[%s] incorrect word count for content '%s'. expected %v, got %v", ext, p.plain, 74, p.WordCount) + } - if p.WordCount != 74 { - t.Fatalf("incorrect word count for content '%s'. expected %v, got %v", p.plain, 74, p.WordCount) - } + if p.Summary != simplePageWithMainEnglishWithCJKRunesSummary { + t.Fatalf("[%s] incorrect Summary for content '%s'. expected %v, got %v", ext, p.plain, + simplePageWithMainEnglishWithCJKRunesSummary, p.Summary) + } - if p.Summary != simplePageWithMainEnglishWithCJKRunesSummary { - t.Fatalf("incorrect Summary for content '%s'. expected %v, got %v", p.plain, - simplePageWithMainEnglishWithCJKRunesSummary, p.Summary) } + + testAllMarkdownEnginesForPage(t, assertFunc, "simple", simplePageWithMainEnglishWithCJKRunes) } func TestWordCountWithIsCJKLanguageFalse(t *testing.T) { testCommonResetState() - viper.Set("HasCJKLanguage", true) - p, _ := NewPage("simple.md") - _, err := p.ReadFrom(strings.NewReader(simplePageWithIsCJKLanguageFalse)) - p.Convert() - p.analyzePage() - if err != nil { - t.Fatalf("Unable to create a page with frontmatter and body content: %s", err) - } + assertFunc := func(t *testing.T, ext string, p *Page) { + if p.WordCount != 75 { + t.Fatalf("[%s] incorrect word count for content '%s'. expected %v, got %v", ext, p.plain, 74, p.WordCount) + } - if p.WordCount != 75 { - t.Fatalf("incorrect word count for content '%s'. expected %v, got %v", p.plain, 75, p.WordCount) - } + if p.Summary != simplePageWithIsCJKLanguageFalseSummary { + t.Fatalf("[%s] incorrect Summary for content '%s'. expected %v, got %v", ext, p.plain, + simplePageWithIsCJKLanguageFalseSummary, p.Summary) + } - if p.Summary != simplePageWithIsCJKLanguageFalseSummary { - t.Fatalf("incorrect Summary for content '%s'. expected %v, got %v", p.plain, - simplePageWithIsCJKLanguageFalseSummary, p.Summary) } + + testAllMarkdownEnginesForPage(t, assertFunc, "simple", simplePageWithIsCJKLanguageFalse) + } func TestWordCount(t *testing.T) { - p, _ := NewPage("simple.md") - _, err := p.ReadFrom(strings.NewReader(simplePageWithLongContent)) - p.Convert() - p.analyzePage() - if err != nil { - t.Fatalf("Unable to create a page with frontmatter and body content: %s", err) - } - if p.WordCount != 483 { - t.Fatalf("incorrect word count. expected %v, got %v", 483, p.WordCount) - } + assertFunc := func(t *testing.T, ext string, p *Page) { + if p.WordCount != 483 { + t.Fatalf("[%s] incorrect word count. expected %v, got %v", ext, 483, p.WordCount) + } - if p.FuzzyWordCount != 500 { - t.Fatalf("incorrect word count. expected %v, got %v", 500, p.WordCount) - } + if p.FuzzyWordCount != 500 { + t.Fatalf("[%s] incorrect word count. expected %v, got %v", ext, 500, p.WordCount) + } + + if p.ReadingTime != 3 { + t.Fatalf("[%s] incorrect min read. expected %v, got %v", ext, 3, p.ReadingTime) + } - if p.ReadingTime != 3 { - t.Fatalf("incorrect min read. expected %v, got %v", 3, p.ReadingTime) + checkTruncation(t, p, true, "long page") } - checkTruncation(t, p, true, "long page") + testAllMarkdownEnginesForPage(t, assertFunc, "simple", simplePageWithLongContent) } func TestCreatePage(t *testing.T) { @@ -1049,15 +1200,18 @@ func TestPageSimpleMethods(t *testing.T) { } func TestChompBOM(t *testing.T) { - p, _ := NewPage("simple.md") const utf8BOM = "\xef\xbb\xbf" - _, err := p.ReadFrom(strings.NewReader(utf8BOM + simplePage)) - p.Convert() - if err != nil { - t.Fatalf("Unable to create a page with BOM prefixed frontmatter and body content: %s", err) + s := newSiteFromSources("simple.md", utf8BOM+simplePage) + + if err := buildSiteSkipRender(s); err != nil { + t.Fatalf("Failed to build site: %s", err) } + require.Len(t, s.Pages, 1) + + p := s.Pages[0] + checkPageTitle(t, p, "Simple") } diff --git a/hugolib/shortcode.go b/hugolib/shortcode.go index 2de00fa9079..b63ba4a4996 100644 --- a/hugolib/shortcode.go +++ b/hugolib/shortcode.go @@ -160,7 +160,11 @@ func HandleShortcodes(stringToParse string, page *Page, t tpl.Template) (string, } if len(tmpShortcodes) > 0 { - tmpContentWithTokensReplaced, err := replaceShortcodeTokens([]byte(tmpContent), shortcodePlaceholderPrefix, tmpShortcodes) + shortcodes, err := executeShortcodeFuncMap(tmpShortcodes) + if err != nil { + return "", err + } + tmpContentWithTokensReplaced, err := replaceShortcodeTokens([]byte(tmpContent), shortcodePlaceholderPrefix, shortcodes) if err != nil { return "", fmt.Errorf("Fail to replace short code tokens in %s:\n%s", page.BaseFileName(), err.Error()) @@ -274,7 +278,7 @@ func renderShortcode(sc shortcode, parent *ShortcodeWithPage, p *Page, t tpl.Tem return renderShortcodeWithPage(tmpl, data) } -func extractAndRenderShortcodes(stringToParse string, p *Page, t tpl.Template) (string, map[string]string, error) { +func extractAndRenderShortcodes(stringToParse string, p *Page, t tpl.Template) (string, map[string]func() (string, error), error) { if p.rendered { panic("Illegal state: Page already marked as rendered, please reuse the shortcodes") @@ -297,15 +301,32 @@ func extractAndRenderShortcodes(stringToParse string, p *Page, t tpl.Template) ( } -func renderShortcodes(shortcodes map[string]shortcode, p *Page, t tpl.Template) map[string]string { - renderedShortcodes := make(map[string]string) +var emptyShortcodeFn = func() (string, error) { return "", nil } + +func executeShortcodeFuncMap(funcs map[string]func() (string, error)) (map[string]string, error) { + result := make(map[string]string) + + for k, v := range funcs { + s, err := v() + if err != nil { + return nil, fmt.Errorf("Failed to execute shortcode with key %s: %s", k, err) + } + result[k] = s + } + + return result, nil +} + +func renderShortcodes(shortcodes map[string]shortcode, p *Page, t tpl.Template) map[string]func() (string, error) { + renderedShortcodes := make(map[string]func() (string, error)) for key, sc := range shortcodes { if sc.err != nil { // need to have something to replace with - renderedShortcodes[key] = "" + renderedShortcodes[key] = emptyShortcodeFn } else { - renderedShortcodes[key] = renderShortcode(sc, nil, p, t) + shorctode := sc + renderedShortcodes[key] = func() (string, error) { return renderShortcode(shorctode, nil, p, t), nil } } } diff --git a/hugolib/shortcode_test.go b/hugolib/shortcode_test.go index 5069fa195c4..dd16bff07d9 100644 --- a/hugolib/shortcode_test.go +++ b/hugolib/shortcode_test.go @@ -484,9 +484,34 @@ e`, {"sect/doc8.rst", `**Shortcodes:** *b: {{< b >}} c: {{% c %}}*`, filepath.FromSlash("sect/doc8/index.html"), "
\n\n\n

Shortcodes: b: b c: c

\n
"}, - {"sect/doc9.mmark", `**Shortcodes:** *b: {{< b >}} c: {{% c %}}*`, + {"sect/doc9.mmark", ` +--- +menu: + main: + parent: 'parent' +--- +**Shortcodes:** *b: {{< b >}} c: {{% c %}}*`, filepath.FromSlash("sect/doc9/index.html"), "

Shortcodes: b: b c: c

\n"}, + // Issue #1229: Menus not available in shortcode. + {"sect/doc10.md", `--- +menu: + main: + identifier: 'parent' +tags: +- Menu +--- +**Menus:** {{< menu >}}`, + filepath.FromSlash("sect/doc10/index.html"), + "

Menus: 1

\n"}, + // Issue #2323: Taxonomies not available in shortcode. + {"sect/doc11.md", `--- +tags: +- Bugs +--- +**Tags:** {{< tags >}}`, + filepath.FromSlash("sect/doc11/index.html"), + "

Tags: 2

\n"}, } sources := make([]source.ByteSource, len(tests)) @@ -507,6 +532,8 @@ e`, templ.AddInternalShortcode("b.html", `b`) templ.AddInternalShortcode("c.html", `c`) templ.AddInternalShortcode("d.html", `d`) + templ.AddInternalShortcode("menu.html", `{{ len (index .Page.Menus "main").Children }}`) + templ.AddInternalShortcode("tags.html", `{{ len .Page.Site.Taxonomies.tags }}`) return nil @@ -540,7 +567,7 @@ e`, content := helpers.ReaderToString(file) if content != test.expected { - t.Errorf("%s content expected:\n%q\ngot:\n%q", test.outFile, test.expected, content) + t.Fatalf("%s content expected:\n%q\ngot:\n%q", test.outFile, test.expected, content) } } diff --git a/hugolib/site.go b/hugolib/site.go index 59b8379dc5d..03e697130bd 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -113,6 +113,26 @@ func newSiteDefaultLang() *Site { return NewSite(newDefaultLanguage()) } +// Convenience func used in tests. +func newSiteFromSources(pathContentPairs ...string) *Site { + if len(pathContentPairs)%2 != 0 { + panic("pathContentPairs must come in pairs") + } + + sources := make([]source.ByteSource, 0) + + for i := 0; i < len(pathContentPairs); i += 2 { + path := pathContentPairs[i] + content := pathContentPairs[i+1] + sources = append(sources, source.ByteSource{Name: filepath.FromSlash(path), Content: []byte(content)}) + } + + return &Site{ + Source: &source.InMemorySource{ByteSource: sources}, + Language: newDefaultLanguage(), + } +} + type targetList struct { page target.Output pageUgly target.Output diff --git a/hugolib/site_test.go b/hugolib/site_test.go index b9e2d346b59..f98bc6a7dec 100644 --- a/hugolib/site_test.go +++ b/hugolib/site_test.go @@ -30,6 +30,7 @@ import ( "github.com/spf13/hugo/target" "github.com/spf13/viper" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) const ( @@ -84,10 +85,16 @@ func pageMust(p *Page, err error) *Page { } func TestDegenerateRenderThingMissingTemplate(t *testing.T) { - p, _ := NewPageFrom(strings.NewReader(pageSimpleTitle), "content/a/file.md") - p.Convert() - s := new(Site) - s.prepTemplates(nil) + s := newSiteFromSources("content/a/file.md", pageSimpleTitle) + + if err := buildSiteSkipRender(s); err != nil { + t.Fatalf("Failed to build site: %s", err) + } + + require.Len(t, s.Pages, 1) + + p := s.Pages[0] + err := s.renderThing(p, "foobar", nil) if err == nil { t.Errorf("Expected err to be returned when missing the template.")