markdown: Pass emoji codes to yuin/goldmark-emoji

Removes emoji code conversion from the page and shortcode parsers. Emoji
codes in markdown are now passed to Goldmark, where the goldmark-emoji
extension converts them to decimal numeric character references.

This disables emoji rendering for the alternate content formats: html,
asciidoc, org, pandoc, and rst.

Fixes #7332
Fixes #11587
Closes #11598

config/allconfig/configlanguage.go

@@ -228,3 +228,7 @@ func (c ConfigLanguage) PaginatePath() string {
 func (c ConfigLanguage) StaticDirs() []string {
 	return c.config.staticDirs()
 }
+
+func (c ConfigLanguage) EnableEmoji() bool {
+	return c.config.EnableEmoji
+}

config/configProvider.go

@@ -66,6 +66,7 @@ type AllProvider interface {
 	StaticDirs() []string
 	IgnoredErrors() map[string]bool
 	WorkingDir() string
+	EnableEmoji() bool
 }
 
 // Provider provides the configuration settings for Hugo.

go.mod

@@ -64,6 +64,7 @@ require (
 	github.com/tdewolff/minify/v2 v2.12.9
 	github.com/tdewolff/parse/v2 v2.6.8
 	github.com/yuin/goldmark v1.5.6
+	github.com/yuin/goldmark-emoji v1.0.2
 	go.uber.org/atomic v1.11.0
 	go.uber.org/automaxprocs v1.5.3
 	gocloud.dev v0.34.0

go.sum

@@ -444,9 +444,12 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.3.7/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/yuin/goldmark v1.5.6 h1:COmQAWTCcGetChm3Ig7G/t8AFAN00t+o8Mt4cf7JpwA=
 github.com/yuin/goldmark v1.5.6/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+github.com/yuin/goldmark-emoji v1.0.2 h1:c/RgTShNgHTtc6xdz2KKI74jJr6rWi7FPgnP9GAsO5s=
+github.com/yuin/goldmark-emoji v1.0.2/go.mod h1:RhP/RWpexdp+KHs7ghKnifRoIs/Bq4nDS7tRbCkOwKY=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=

helpers/emoji_test.go

@@ -13,13 +13,8 @@
 package helpers
 
 import (
-	"math"
 	"reflect"
-	"strings"
 	"testing"
-
-	"github.com/gohugoio/hugo/bufferpool"
-	"github.com/kyokomi/emoji/v2"
 )
 
 func TestEmojiCustom(t *testing.T) {
@@ -68,76 +63,3 @@ func TestEmojiCustom(t *testing.T) {
 
 	}
 }
-
-// The Emoji benchmarks below are heavily skewed in Hugo's direction:
-//
-// Hugo have a byte slice, wants a byte slice and doesn't mind if the original is modified.
-
-func BenchmarkEmojiKyokomiFprint(b *testing.B) {
-	f := func(in []byte) []byte {
-		buff := bufferpool.GetBuffer()
-		defer bufferpool.PutBuffer(buff)
-		emoji.Fprint(buff, string(in))
-
-		bc := make([]byte, buff.Len())
-		copy(bc, buff.Bytes())
-		return bc
-	}
-
-	doBenchmarkEmoji(b, f)
-}
-
-func BenchmarkEmojiKyokomiSprint(b *testing.B) {
-	f := func(in []byte) []byte {
-		return []byte(emoji.Sprint(string(in)))
-	}
-
-	doBenchmarkEmoji(b, f)
-}
-
-func BenchmarkHugoEmoji(b *testing.B) {
-	doBenchmarkEmoji(b, Emojify)
-}
-
-func doBenchmarkEmoji(b *testing.B, f func(in []byte) []byte) {
-	type input struct {
-		in     []byte
-		expect []byte
-	}
-
-	data := []struct {
-		input  string
-		expect string
-	}{
-		{"A :smile: a day", emoji.Sprint("A :smile: a day")},
-		{"A :smile: and a :beer: day keeps the doctor away", emoji.Sprint("A :smile: and a :beer: day keeps the doctor away")},
-		{"A :smile: a day and 10 " + strings.Repeat(":beer: ", 10), emoji.Sprint("A :smile: a day and 10 " + strings.Repeat(":beer: ", 10))},
-		{"No smiles today.", "No smiles today."},
-		{"No smiles for you or " + strings.Repeat("you ", 1000), "No smiles for you or " + strings.Repeat("you ", 1000)},
-	}
-
-	in := make([]input, b.N*len(data))
-	cnt := 0
-	for i := 0; i < b.N; i++ {
-		for _, this := range data {
-			in[cnt] = input{[]byte(this.input), []byte(this.expect)}
-			cnt++
-		}
-	}
-
-	b.ResetTimer()
-	cnt = 0
-	for i := 0; i < b.N; i++ {
-		for j := range data {
-			currIn := in[cnt]
-			cnt++
-			result := f(currIn.in)
-			// The Emoji implementations gives slightly different output.
-			diffLen := len(result) - len(currIn.expect)
-			diffLen = int(math.Abs(float64(diffLen)))
-			if diffLen > 30 {
-				b.Fatalf("[%d] emoji std, got \n%q but expected \n%q", j, result, currIn.expect)
-			}
-		}
-	}
-}

hugolib/content_map_page.go

@@ -150,7 +150,7 @@ func (m *pageMap) newPageFromContentNode(n *contentNode, parentBucket *pagesMapB
 
 	parseResult, err := pageparser.Parse(
 		r,
-		pageparser.Config{EnableEmoji: s.conf.EnableEmoji},
+		pageparser.Config{},
 	)
 	if err != nil {
 		return nil, err

hugolib/page.go

@@ -794,11 +794,7 @@ Loop:
 			rn.AddShortcode(currShortcode)
 
 		case it.Type == pageparser.TypeEmoji:
-			if emoji := helpers.Emoji(it.ValStr(result.Input())); emoji != nil {
-				rn.AddReplacement(emoji, it)
-			} else {
-				rn.AddBytes(it)
-			}
+			rn.AddBytes(it)
 		case it.IsEOF():
 			break Loop
 		case it.IsError():

hugolib/page_test.go

@@ -894,13 +894,13 @@ summary: Summary (zh)
 	b.Build(BuildCfg{})
 
 	b.AssertFileContent("public/index.html", `<html>
-	
+
 <body>
-	
+
 <h2>Translations</h2>
 <ul>
 
-	
+
 <li>Title: Title (zh), Summary (zh)</li>
 <li>Content: <p>这是一些内容</p>
 </li>
@@ -911,7 +911,7 @@ summary: Summary (zh)
 <li>Truncated: false</li>
 <li>FuzzyWordCount: 100</li>
 <li>ReadingTime: 1</li>
-<li>Len: 26</li>	
+<li>Len: 26</li>
 
 </ul>
 
@@ -921,7 +921,7 @@ summary: Summary (zh)
 	b.AssertFileContent("public/metadata.html", `<h2>Translations metadata</h2>
 <ul>
 
-	
+
 <li>Title: Title (zh), Summary (zh)</li>
 <li>Content: <p>这是一些内容</p>
 </li>
@@ -932,13 +932,13 @@ summary: Summary (zh)
 <li>Truncated: false</li>
 <li>FuzzyWordCount: 100</li>
 <li>ReadingTime: 1</li>
-<li>Len: 26</li>	
+<li>Len: 26</li>
 
 </ul>`)
 	b.AssertFileContent("public/zh_cn/index.html", `<html>
 
 <body>
-	
+
 <h2>Translations</h2>
 <ul>
 
@@ -953,7 +953,7 @@ summary: Summary (zh)
 <li>Truncated: false</li>
 <li>FuzzyWordCount: 100</li>
 <li>ReadingTime: 1</li>
-<li>Len: 29</li>	
+<li>Len: 29</li>
 
 </ul>
 
@@ -963,7 +963,7 @@ summary: Summary (zh)
 	b.AssertFileContent("public/zh_cn/metadata.html", `<h2>Translations metadata</h2>
 <ul>
 
-	
+
 <li>Title: Title (en), Summary (en)</li>
 <li>Content: <p>Here is some content.</p>
 </li>
@@ -974,7 +974,7 @@ summary: Summary (zh)
 <li>Truncated: false</li>
 <li>FuzzyWordCount: 100</li>
 <li>ReadingTime: 1</li>
-<li>Len: 29</li>	
+<li>Len: 29</li>
 
 </ul>`)
 }
@@ -1234,60 +1234,6 @@ func TestChompBOM(t *testing.T) {
 	checkPageTitle(t, p, "Simple")
 }
 
-func TestPageWithEmoji(t *testing.T) {
-	for _, enableEmoji := range []bool{true, false} {
-		v := config.New()
-		v.Set("enableEmoji", enableEmoji)
-
-		b := newTestSitesBuilder(t).WithViper(v)
-
-		b.WithContent("page-emoji.md", `---
-title: "Hugo Smile"
----
-This is a :smile:.
-<!--more--> 
-
-Another :smile: This is :not: :an: :emoji:.
-
-O :christmas_tree:
-
-Write me an :e-mail: or :email:?
-
-Too many colons: :: ::: :::: :?: :!: :.:
-
-If you dislike this video, you can hit that :-1: button :stuck_out_tongue_winking_eye:,
-but if you like it, hit :+1: and get subscribed!
-`)
-
-		b.CreateSites().Build(BuildCfg{})
-
-		if enableEmoji {
-			b.AssertFileContent("public/page-emoji/index.html",
-				"This is a 😄",
-				"Another 😄",
-				"This is :not: :an: :emoji:.",
-				"O 🎄",
-				"Write me an 📧 or ✉️?",
-				"Too many colons: :: ::: :::: :?: :!: :.:",
-				"you can hit that 👎 button 😜,",
-				"hit 👍 and get subscribed!",
-			)
-		} else {
-			b.AssertFileContent("public/page-emoji/index.html",
-				"This is a :smile:",
-				"Another :smile:",
-				"This is :not: :an: :emoji:.",
-				"O :christmas_tree:",
-				"Write me an :e-mail: or :email:?",
-				"Too many colons: :: ::: :::: :?: :!: :.:",
-				"you can hit that :-1: button :stuck_out_tongue_winking_eye:,",
-				"hit :+1: and get subscribed!",
-			)
-		}
-
-	}
-}
-
 func TestPageHTMLContent(t *testing.T) {
 	b := newTestSitesBuilder(t)
 	b.WithSimpleConfigFile()
@@ -1333,7 +1279,7 @@ func TestPageManualSummary(t *testing.T) {
 title: "Hugo"
 ---
 This is a {{< sc >}}.
-<!--more--> 
+<!--more-->
 Content.
 `)
 
@@ -1342,7 +1288,7 @@ Content.
 title: "Hugo"
 ---
 {{< sc >}}
-<!--more--> 
+<!--more-->
 {{< sc >}}
 `)
 
@@ -1363,15 +1309,15 @@ Summary<!--more-->{{< sc >}}
 #+DESCRIPTION: D1
 This is a {{< sc >}}.
 # more
-Content.	
+Content.
 `)
 
 	b.WithContent("page-org-variant1.org", `#+TITLE: T1
 Summary.
 
 # more
 
-Content.	
+Content.
 `)
 
 	b.WithTemplatesAdded("layouts/shortcodes/sc.html", "a shortcode")
@@ -1665,7 +1611,7 @@ SUMMARY:{{ .Summary }}:{{ len .Summary }}:END
 
 	b := newTestSitesBuilder(t)
 	b.WithSimpleConfigFile().WithTemplatesAdded(single...).WithContent("p1.md", fmt.Sprintf(`---
-title: p1	
+title: p1
 ---
 
 %s

hugolib/shortcode.go

@@ -26,8 +26,6 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/gohugoio/hugo/helpers"
-
 	"errors"
 
 	"github.com/gohugoio/hugo/common/herrors"
@@ -632,14 +630,7 @@ Loop:
 		case currItem.IsText():
 			sc.inner = append(sc.inner, currItem.ValStr(source))
 		case currItem.Type == pageparser.TypeEmoji:
-			// TODO(bep) avoid the duplication of these "text cases", to prevent
-			// more of #6504 in the future.
-			val := currItem.ValStr(source)
-			if emoji := helpers.Emoji(val); emoji != nil {
-				sc.inner = append(sc.inner, string(emoji))
-			} else {
-				sc.inner = append(sc.inner, val)
-			}
+			sc.inner = append(sc.inner, currItem.ValStr(source))
 		case currItem.IsShortcodeName():
 
 			sc.name = currItem.ValStr(source)