Permalink
Browse files

Move the emoji parsing to pageparser

This avoids double parsing the page content when `enableEmoji=true`.

This commit also adds some general improvements to the parser, making it in general much faster:

```bash
benchmark                     old ns/op     new ns/op     delta
BenchmarkShortcodeLexer-4     90258         101730        +12.71%
BenchmarkParse-4              148940        15037         -89.90%

benchmark                     old allocs     new allocs     delta
BenchmarkShortcodeLexer-4     456            700            +53.51%
BenchmarkParse-4              28             33             +17.86%

benchmark                     old bytes     new bytes     delta
BenchmarkShortcodeLexer-4     69875         81014         +15.94%
BenchmarkParse-4              8128          8304          +2.17%
```

Running some site benchmarks with Emoji support turned on:

```bash
benchmark                                                                                     old ns/op     new ns/op     delta
BenchmarkSiteBuilding/TOML,num_langs=3,num_pages=5000,tags_per_page=5,shortcodes,render-4     924556797     818115620     -11.51%

benchmark                                                                                     old allocs     new allocs     delta
BenchmarkSiteBuilding/TOML,num_langs=3,num_pages=5000,tags_per_page=5,shortcodes,render-4     4112613        4133787        +0.51%

benchmark                                                                                     old bytes     new bytes     delta
BenchmarkSiteBuilding/TOML,num_langs=3,num_pages=5000,tags_per_page=5,shortcodes,render-4     426982864     424363832     -0.61%
```

Fixes #5534
  • Loading branch information...
bep committed Dec 17, 2018
1 parent a8853f1 commit 9cd54cab20a03475e34ca462bd943069111481ae
@@ -215,7 +215,7 @@ type parsedFile struct {
func parseContentFile(r io.Reader) (parsedFile, error) {
var pf parsedFile

psr, err := pageparser.Parse(r)
psr, err := pageparser.Parse(r, pageparser.Config{})
if err != nil {
return pf, err
}
@@ -30,6 +30,12 @@ var (
emojiMaxSize int
)

// Emoji returns the emojy given a key, e.g. ":smile:", nil if not found.
func Emoji(key string) []byte {
emojiInit.Do(initEmoji)
return emojis[key]
}

// Emojify "emojifies" the input source.
// Note that the input byte slice will be modified if needed.
// See http://www.emoji-cheat-sheet.com/
@@ -17,6 +17,8 @@ import (
"bytes"
"io"

"github.com/gohugoio/hugo/helpers"

errors "github.com/pkg/errors"

bp "github.com/gohugoio/hugo/bufferpool"
@@ -149,6 +151,12 @@ Loop:
result.WriteString(placeHolder)
ordinal++
s.shortcodes.Add(placeHolder, currShortcode)
case it.Type == pageparser.TypeEmoji:
if emoji := helpers.Emoji(it.ValStr()); emoji != nil {
result.Write(emoji)
} else {
result.Write(it.Val)
}
case it.IsEOF():
break Loop
case it.IsError():
@@ -170,7 +178,10 @@ Loop:

func (p *Page) parse(reader io.Reader) error {

parseResult, err := pageparser.Parse(reader)
parseResult, err := pageparser.Parse(
reader,
pageparser.Config{EnableEmoji: p.s.Cfg.GetBool("enableEmoji")},
)
if err != nil {
return err
}
@@ -1497,6 +1497,45 @@ func TestChompBOM(t *testing.T) {
checkPageTitle(t, p, "Simple")
}

func TestPageWithEmoji(t *testing.T) {
for _, enableEmoji := range []bool{true, false} {
v := viper.New()
v.Set("enableEmoji", enableEmoji)
b := newTestSitesBuilder(t)
b.WithViper(v)

b.WithSimpleConfigFile()

b.WithContent("page-emoji.md", `---
title: "Hugo Smile"
---
This is a :smile:.
<!--more-->
Another :smile: This is :not: an emoji.
`)

b.CreateSites().Build(BuildCfg{})

if enableEmoji {
b.AssertFileContent("public/page-emoji/index.html",
"This is a 😄",
"Another 😄",
"This is :not: an emoji",
)
} else {
b.AssertFileContent("public/page-emoji/index.html",
"This is a :smile:",
"Another :smile:",
"This is :not: an emoji",
)
}

}

}

// https://github.com/gohugoio/hugo/issues/5381
func TestPageManualSummary(t *testing.T) {
b := newTestSitesBuilder(t)
@@ -272,10 +272,6 @@ func (c *contentHandlers) handlePageContent() contentHandler {

p := ctx.currentPage

if c.s.Cfg.GetBool("enableEmoji") {
p.workContent = helpers.Emojify(p.workContent)
}

p.workContent = p.renderContent(p.workContent)

tmpContent, tmpTableOfContents := helpers.ExtractTOC(p.workContent)
@@ -177,6 +177,16 @@ type shortcode struct {
pos int // the position in bytes in the source file
}

func (s shortcode) innerString() string {
var sb strings.Builder

for _, inner := range s.inner {
sb.WriteString(inner.(string))
}

return sb.String()
}

func (sc shortcode) String() string {
// for testing (mostly), so any change here will break tests!
var params interface{}
@@ -363,7 +373,7 @@ func renderShortcode(
if sc.isInline {
templName := path.Join("_inline_shortcode", p.Path(), sc.name)
if sc.isClosing {
templStr := sc.inner[0].(string)
templStr := sc.innerString()

var err error
tmpl, err = p.s.TextTmpl.Parse(templName, templStr)
@@ -113,6 +113,7 @@ const (
TypeFrontMatterTOML
TypeFrontMatterJSON
TypeFrontMatterORG
TypeEmoji
TypeIgnore // // The BOM Unicode byte order marker and possibly others

// shortcode items
Oops, something went wrong.

0 comments on commit 9cd54ca

Please sign in to comment.