diff --git a/config/allconfig/configlanguage.go b/config/allconfig/configlanguage.go index 27ba00d82..2c5a116f4 100644 --- a/config/allconfig/configlanguage.go +++ b/config/allconfig/configlanguage.go @@ -228,3 +228,7 @@ func (c ConfigLanguage) PaginatePath() string { func (c ConfigLanguage) StaticDirs() []string { return c.config.staticDirs() } + +func (c ConfigLanguage) EnableEmoji() bool { + return c.config.EnableEmoji +} diff --git a/config/configProvider.go b/config/configProvider.go index 946830056..8e2ab0334 100644 --- a/config/configProvider.go +++ b/config/configProvider.go @@ -66,6 +66,7 @@ type AllProvider interface { StaticDirs() []string IgnoredErrors() map[string]bool WorkingDir() string + EnableEmoji() bool } // Provider provides the configuration settings for Hugo. diff --git a/go.mod b/go.mod index 29c25cdb4..189ce80b5 100644 --- a/go.mod +++ b/go.mod @@ -64,6 +64,7 @@ require ( github.com/tdewolff/minify/v2 v2.12.9 github.com/tdewolff/parse/v2 v2.6.8 github.com/yuin/goldmark v1.5.6 + github.com/yuin/goldmark-emoji v1.0.2 go.uber.org/atomic v1.11.0 go.uber.org/automaxprocs v1.5.3 gocloud.dev v0.34.0 diff --git a/go.sum b/go.sum index 9cb52433b..baa0ebadf 100644 --- a/go.sum +++ b/go.sum @@ -444,9 +444,12 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.3.7/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark v1.5.6 h1:COmQAWTCcGetChm3Ig7G/t8AFAN00t+o8Mt4cf7JpwA= github.com/yuin/goldmark v1.5.6/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/goldmark-emoji v1.0.2 h1:c/RgTShNgHTtc6xdz2KKI74jJr6rWi7FPgnP9GAsO5s= +github.com/yuin/goldmark-emoji v1.0.2/go.mod h1:RhP/RWpexdp+KHs7ghKnifRoIs/Bq4nDS7tRbCkOwKY= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= diff --git a/helpers/emoji_test.go b/helpers/emoji_test.go index b45444d45..c6a844b9d 100644 --- a/helpers/emoji_test.go +++ b/helpers/emoji_test.go @@ -13,13 +13,8 @@ package helpers import ( - "math" "reflect" - "strings" "testing" - - "github.com/gohugoio/hugo/bufferpool" - "github.com/kyokomi/emoji/v2" ) func TestEmojiCustom(t *testing.T) { @@ -68,76 +63,3 @@ func TestEmojiCustom(t *testing.T) { } } - -// The Emoji benchmarks below are heavily skewed in Hugo's direction: -// -// Hugo have a byte slice, wants a byte slice and doesn't mind if the original is modified. - -func BenchmarkEmojiKyokomiFprint(b *testing.B) { - f := func(in []byte) []byte { - buff := bufferpool.GetBuffer() - defer bufferpool.PutBuffer(buff) - emoji.Fprint(buff, string(in)) - - bc := make([]byte, buff.Len()) - copy(bc, buff.Bytes()) - return bc - } - - doBenchmarkEmoji(b, f) -} - -func BenchmarkEmojiKyokomiSprint(b *testing.B) { - f := func(in []byte) []byte { - return []byte(emoji.Sprint(string(in))) - } - - doBenchmarkEmoji(b, f) -} - -func BenchmarkHugoEmoji(b *testing.B) { - doBenchmarkEmoji(b, Emojify) -} - -func doBenchmarkEmoji(b *testing.B, f func(in []byte) []byte) { - type input struct { - in []byte - expect []byte - } - - data := []struct { - input string - expect string - }{ - {"A :smile: a day", emoji.Sprint("A :smile: a day")}, - {"A :smile: and a :beer: day keeps the doctor away", emoji.Sprint("A :smile: and a :beer: day keeps the doctor away")}, - {"A :smile: a day and 10 " + strings.Repeat(":beer: ", 10), emoji.Sprint("A :smile: a day and 10 " + strings.Repeat(":beer: ", 10))}, - {"No smiles today.", "No smiles today."}, - {"No smiles for you or " + strings.Repeat("you ", 1000), "No smiles for you or " + strings.Repeat("you ", 1000)}, - } - - in := make([]input, b.N*len(data)) - cnt := 0 - for i := 0; i < b.N; i++ { - for _, this := range data { - in[cnt] = input{[]byte(this.input), []byte(this.expect)} - cnt++ - } - } - - b.ResetTimer() - cnt = 0 - for i := 0; i < b.N; i++ { - for j := range data { - currIn := in[cnt] - cnt++ - result := f(currIn.in) - // The Emoji implementations gives slightly different output. - diffLen := len(result) - len(currIn.expect) - diffLen = int(math.Abs(float64(diffLen))) - if diffLen > 30 { - b.Fatalf("[%d] emoji std, got \n%q but expected \n%q", j, result, currIn.expect) - } - } - } -} diff --git a/hugolib/content_map_page.go b/hugolib/content_map_page.go index e7f6c5aed..2c14ffa59 100644 --- a/hugolib/content_map_page.go +++ b/hugolib/content_map_page.go @@ -150,7 +150,7 @@ func (m *pageMap) newPageFromContentNode(n *contentNode, parentBucket *pagesMapB parseResult, err := pageparser.Parse( r, - pageparser.Config{EnableEmoji: s.conf.EnableEmoji}, + pageparser.Config{}, ) if err != nil { return nil, err diff --git a/hugolib/page.go b/hugolib/page.go index 5b2411441..48f4c9d2b 100644 --- a/hugolib/page.go +++ b/hugolib/page.go @@ -794,11 +794,7 @@ Loop: rn.AddShortcode(currShortcode) case it.Type == pageparser.TypeEmoji: - if emoji := helpers.Emoji(it.ValStr(result.Input())); emoji != nil { - rn.AddReplacement(emoji, it) - } else { - rn.AddBytes(it) - } + rn.AddBytes(it) case it.IsEOF(): break Loop case it.IsError(): diff --git a/hugolib/page_test.go b/hugolib/page_test.go index ecaf1dc5c..7e34f0499 100644 --- a/hugolib/page_test.go +++ b/hugolib/page_test.go @@ -894,13 +894,13 @@ summary: Summary (zh) b.Build(BuildCfg{}) b.AssertFileContent("public/index.html", ` - + - +

Translations

@@ -921,7 +921,7 @@ summary: Summary (zh) b.AssertFileContent("public/metadata.html", `

Translations metadata

`) b.AssertFileContent("public/zh_cn/index.html", ` - +

Translations

@@ -963,7 +963,7 @@ summary: Summary (zh) b.AssertFileContent("public/zh_cn/metadata.html", `

Translations metadata

`) } @@ -1234,60 +1234,6 @@ func TestChompBOM(t *testing.T) { checkPageTitle(t, p, "Simple") } -func TestPageWithEmoji(t *testing.T) { - for _, enableEmoji := range []bool{true, false} { - v := config.New() - v.Set("enableEmoji", enableEmoji) - - b := newTestSitesBuilder(t).WithViper(v) - - b.WithContent("page-emoji.md", `--- -title: "Hugo Smile" ---- -This is a :smile:. - - -Another :smile: This is :not: :an: :emoji:. - -O :christmas_tree: - -Write me an :e-mail: or :email:? - -Too many colons: :: ::: :::: :?: :!: :.: - -If you dislike this video, you can hit that :-1: button :stuck_out_tongue_winking_eye:, -but if you like it, hit :+1: and get subscribed! -`) - - b.CreateSites().Build(BuildCfg{}) - - if enableEmoji { - b.AssertFileContent("public/page-emoji/index.html", - "This is a 😄", - "Another 😄", - "This is :not: :an: :emoji:.", - "O 🎄", - "Write me an 📧 or ✉️?", - "Too many colons: :: ::: :::: :?: :!: :.:", - "you can hit that 👎 button 😜,", - "hit 👍 and get subscribed!", - ) - } else { - b.AssertFileContent("public/page-emoji/index.html", - "This is a :smile:", - "Another :smile:", - "This is :not: :an: :emoji:.", - "O :christmas_tree:", - "Write me an :e-mail: or :email:?", - "Too many colons: :: ::: :::: :?: :!: :.:", - "you can hit that :-1: button :stuck_out_tongue_winking_eye:,", - "hit :+1: and get subscribed!", - ) - } - - } -} - func TestPageHTMLContent(t *testing.T) { b := newTestSitesBuilder(t) b.WithSimpleConfigFile() @@ -1333,7 +1279,7 @@ func TestPageManualSummary(t *testing.T) { title: "Hugo" --- This is a {{< sc >}}. - + Content. `) @@ -1342,7 +1288,7 @@ Content. title: "Hugo" --- {{< sc >}} - + {{< sc >}} `) @@ -1363,7 +1309,7 @@ Summary{{< sc >}} #+DESCRIPTION: D1 This is a {{< sc >}}. # more -Content. +Content. `) b.WithContent("page-org-variant1.org", `#+TITLE: T1 @@ -1371,7 +1317,7 @@ Summary. # more -Content. +Content. `) b.WithTemplatesAdded("layouts/shortcodes/sc.html", "a shortcode") @@ -1665,7 +1611,7 @@ SUMMARY:{{ .Summary }}:{{ len .Summary }}:END b := newTestSitesBuilder(t) b.WithSimpleConfigFile().WithTemplatesAdded(single...).WithContent("p1.md", fmt.Sprintf(`--- -title: p1 +title: p1 --- %s diff --git a/hugolib/shortcode.go b/hugolib/shortcode.go index e201d4d6b..8edfab118 100644 --- a/hugolib/shortcode.go +++ b/hugolib/shortcode.go @@ -26,8 +26,6 @@ import ( "strings" "sync" - "github.com/gohugoio/hugo/helpers" - "errors" "github.com/gohugoio/hugo/common/herrors" @@ -632,14 +630,7 @@ Loop: case currItem.IsText(): sc.inner = append(sc.inner, currItem.ValStr(source)) case currItem.Type == pageparser.TypeEmoji: - // TODO(bep) avoid the duplication of these "text cases", to prevent - // more of #6504 in the future. - val := currItem.ValStr(source) - if emoji := helpers.Emoji(val); emoji != nil { - sc.inner = append(sc.inner, string(emoji)) - } else { - sc.inner = append(sc.inner, val) - } + sc.inner = append(sc.inner, currItem.ValStr(source)) case currItem.IsShortcodeName(): sc.name = currItem.ValStr(source) diff --git a/hugolib/shortcode_test.go b/hugolib/shortcode_test.go index 6ee68673f..ff9ae35fc 100644 --- a/hugolib/shortcode_test.go +++ b/hugolib/shortcode_test.go @@ -599,7 +599,7 @@ weight: %d c.Assert(len(s.RegularPages()), qt.Equals, 3) builder.AssertFileContent("public/en/p1/index.html", `v1: 0 sgo: |v2: 1 sgo: 0|v3: 2 sgo: 1|v4: 3 sgo: 2|v5: 4 sgo: 3`) - builder.AssertFileContent("public/en/p1/index.html", `outer ordinal: 5 inner: + builder.AssertFileContent("public/en/p1/index.html", `outer ordinal: 5 inner: ordinal: 0 scratch ordinal: 1 scratch get ordinal: 0 ordinal: 2 scratch ordinal: 3 scratch get ordinal: 2 ordinal: 4 scratch ordinal: 5 scratch get ordinal: 4`) @@ -754,33 +754,6 @@ title: "Hugo Rocks!" ) } -// https://github.com/gohugoio/hugo/issues/6504 -func TestShortcodeEmoji(t *testing.T) { - t.Parallel() - - v := config.New() - v.Set("enableEmoji", true) - - builder := newTestSitesBuilder(t).WithViper(v) - - builder.WithContent("page.md", `--- -title: "Hugo Rocks!" ---- - -# doc - -{{< event >}}10:30-11:00 My :smile: Event {{< /event >}} - - -`).WithTemplatesAdded( - "layouts/shortcodes/event.html", `
{{ "\u29BE" }} {{ .Inner }}
`) - - builder.Build(BuildCfg{}) - builder.AssertFileContent("public/page/index.html", - "⦾ 10:30-11:00 My 😄 Event", - ) -} - func TestShortcodeParams(t *testing.T) { t.Parallel() c := qt.New(t) diff --git a/hugolib/site.go b/hugolib/site.go index 863b5c4b7..c682eebc9 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -94,7 +94,6 @@ type siteConfigHolder struct { taxonomiesConfig taxonomiesConfig timeout time.Duration hasCJKLanguage bool - enableEmoji bool } // Lazily loaded site dependencies. diff --git a/markup/goldmark/convert.go b/markup/goldmark/convert.go index fa2ab548b..d66687783 100644 --- a/markup/goldmark/convert.go +++ b/markup/goldmark/convert.go @@ -28,6 +28,7 @@ import ( "github.com/gohugoio/hugo/markup/converter" "github.com/gohugoio/hugo/markup/tableofcontents" "github.com/yuin/goldmark" + emoji "github.com/yuin/goldmark-emoji" "github.com/yuin/goldmark/ast" "github.com/yuin/goldmark/extension" "github.com/yuin/goldmark/parser" @@ -149,6 +150,10 @@ func newMarkdown(pcfg converter.ProviderConfig) goldmark.Markdown { extensions = append(extensions, c) } + if pcfg.Conf.EnableEmoji() { + extensions = append(extensions, emoji.Emoji) + } + if cfg.Parser.AutoHeadingID { parserOptions = append(parserOptions, parser.WithAutoHeadingID()) } @@ -156,6 +161,7 @@ func newMarkdown(pcfg converter.ProviderConfig) goldmark.Markdown { if cfg.Parser.Attribute.Title { parserOptions = append(parserOptions, parser.WithAttribute()) } + if cfg.Parser.Attribute.Block { extensions = append(extensions, attributes.New()) } diff --git a/markup/goldmark/integration_test.go b/markup/goldmark/integration_test.go index 84617b2c8..fdcbf4975 100644 --- a/markup/goldmark/integration_test.go +++ b/markup/goldmark/integration_test.go @@ -410,7 +410,7 @@ func TestHookInfiniteRecursion(t *testing.T) { files := ` -- config.toml -- -- layouts/_default/_markup/render-link.html -- -{{ .Text | RENDERFUNC }} +{{ .Text | RENDERFUNC }} -- layouts/_default/single.html -- {{ .Content }} -- content/p1.md -- @@ -421,8 +421,8 @@ title: "p1" https://example.org a@b.com - - + + ` files = strings.ReplaceAll(files, "RENDERFUNC", renderFunc) @@ -578,3 +578,114 @@ a c "
  • This is a list item
    • ", ) } + +// Issue #7332 +// Issue #11587 +func TestGoldmarkEmojiExtension(t *testing.T) { + t.Parallel() + + files := ` +-- config.toml -- +enableEmoji = true +-- content/p1.md -- +--- +title: "p1" +--- +~~~text +:x: +~~~ + +{{% include "/p2" %}} + +{{< sc1 >}}:smiley:{{< /sc1 >}} + +{{< sc2 >}}:+1:{{< /sc2 >}} + +{{% sc3 %}}:-1:{{% /sc3 %}} + +-- content/p2.md -- +--- +title: "p2" +--- +:heavy_check_mark: +-- layouts/shortcodes/include.html -- +{{ $p := site.GetPage (.Get 0) }} +{{ $p.RenderShortcodes }} +-- layouts/shortcodes/sc1.html -- +sc1_begin|{{ .Inner }}|sc1_end +-- layouts/shortcodes/sc2.html -- +sc2_begin|{{ .Inner | .Page.RenderString }}|sc2_end +-- layouts/shortcodes/sc3.html -- +sc3_begin|{{ .Inner }}|sc3_end +-- layouts/_default/single.html -- +{{ .Content }} +` + + b := hugolib.NewIntegrationTestBuilder( + hugolib.IntegrationTestConfig{ + T: t, + TxtarString: files, + }, + ).Build() + + b.AssertFileContentExact("public/p1/index.html", + // Issue #7332 + ":x:\n", + // Issue #11587 + "

    ✔️

    ", + // Should not be converted to emoji + "sc1_begin|:smiley:|sc1_end", + // Should be converted to emoji + "sc2_begin|👍|sc2_end", + // Should be converted to emoji + "sc3_begin|👎|sc3_end", + ) +} + +func TestEmojiDisabled(t *testing.T) { + t.Parallel() + + files := ` +-- config.toml -- +enableEmoji = false +-- content/p1.md -- +--- +title: "p1" +--- +:x: +-- layouts/_default/single.html -- +{{ .Content }} +` + + b := hugolib.NewIntegrationTestBuilder( + hugolib.IntegrationTestConfig{ + T: t, + TxtarString: files, + }, + ).Build() + + b.AssertFileContentExact("public/p1/index.html", "

    :x:

    ") +} + +func TestEmojiDefaultConfig(t *testing.T) { + t.Parallel() + + files := ` +-- content/p1.md -- +--- +title: "p1" +--- +:x: +-- layouts/_default/single.html -- +{{ .Content }} +` + + b := hugolib.NewIntegrationTestBuilder( + hugolib.IntegrationTestConfig{ + T: t, + TxtarString: files, + }, + ).Build() + + b.AssertFileContentExact("public/p1/index.html", "

    :x:

    ") +} diff --git a/parser/pageparser/pagelexer.go b/parser/pageparser/pagelexer.go index 5f5d147e6..64cd4bfc1 100644 --- a/parser/pageparser/pagelexer.go +++ b/parser/pageparser/pagelexer.go @@ -61,9 +61,7 @@ func (l *pageLexer) Input() []byte { return l.input } -type Config struct { - EnableEmoji bool -} +type Config struct{} // note: the input position here is normally 0 (start), but // can be set if position of first shortcode is known @@ -103,8 +101,6 @@ var ( delimOrg = []byte("#+") htmlCommentStart = []byte("") - - emojiDelim = byte(':') ) func (l *pageLexer) next() rune { @@ -276,34 +272,6 @@ func (l *pageLexer) consumeSpace() { } } -// lex a string starting at ":" -func lexEmoji(l *pageLexer) stateFunc { - pos := l.pos + 1 - valid := false - - for i := pos; i < len(l.input); i++ { - if i > pos && l.input[i] == emojiDelim { - pos = i + 1 - valid = true - break - } - r, _ := utf8.DecodeRune(l.input[i:]) - if !(isAlphaNumericOrHyphen(r) || r == '+') { - break - } - } - - if valid { - l.pos = pos - l.emit(TypeEmoji) - } else { - l.pos++ - l.emit(tText) - } - - return lexMainSection -} - type sectionHandlers struct { l *pageLexer @@ -399,20 +367,6 @@ func createSectionHandlers(l *pageLexer) *sectionHandlers { handlers := []*sectionHandler{shortCodeHandler, summaryDividerHandler} - if l.cfg.EnableEmoji { - emojiHandler := §ionHandler{ - l: l, - skipFunc: func(l *pageLexer) int { - return l.indexByte(emojiDelim) - }, - lexFunc: func(origin stateFunc, l *pageLexer) (stateFunc, bool) { - return lexEmoji, true - }, - } - - handlers = append(handlers, emojiHandler) - } - return §ionHandlers{ l: l, handlers: handlers, diff --git a/parser/pageparser/pageparser_main_test.go b/parser/pageparser/pageparser_main_test.go deleted file mode 100644 index 4e3fe8e84..000000000 --- a/parser/pageparser/pageparser_main_test.go +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2018 The Hugo Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pageparser - -import ( - "testing" - - qt "github.com/frankban/quicktest" -) - -func TestMain(t *testing.T) { - t.Parallel() - c := qt.New(t) - - mainTests := []lexerTest{ - {"emoji #1", "Some text with :emoji:", []typeText{nti(tText, "Some text with "), nti(TypeEmoji, ":emoji:"), tstEOF}}, - {"emoji #2", "Some text with :emoji: and some text.", []typeText{nti(tText, "Some text with "), nti(TypeEmoji, ":emoji:"), nti(tText, " and some text."), tstEOF}}, - {"looks like an emoji #1", "Some text and then :emoji", []typeText{nti(tText, "Some text and then "), nti(tText, ":"), nti(tText, "emoji"), tstEOF}}, - {"looks like an emoji #2", "Some text and then ::", []typeText{nti(tText, "Some text and then "), nti(tText, ":"), nti(tText, ":"), tstEOF}}, - {"looks like an emoji #3", ":Some :text", []typeText{nti(tText, ":"), nti(tText, "Some "), nti(tText, ":"), nti(tText, "text"), tstEOF}}, - } - - for i, test := range mainTests { - items := collectWithConfig([]byte(test.input), false, lexMainSection, Config{EnableEmoji: true}) - if !equal(test.input, items, test.items) { - got := itemsToString(items, []byte(test.input)) - expected := testItemsToString(test.items) - c.Assert(got, qt.Equals, expected, qt.Commentf("Test %d: %s", i, test.name)) - } - } -} diff --git a/parser/pageparser/pageparser_test.go b/parser/pageparser/pageparser_test.go index de817d1fb..c58018f0e 100644 --- a/parser/pageparser/pageparser_test.go +++ b/parser/pageparser/pageparser_test.go @@ -24,7 +24,7 @@ import ( func BenchmarkParse(b *testing.B) { start := ` - + --- title: "Front Matters" @@ -38,33 +38,7 @@ This is some summary. This is some summary. This is some summary. This is some s ` input := []byte(start + strings.Repeat(strings.Repeat("this is text", 30)+"{{< myshortcode >}}This is some inner content.{{< /myshortcode >}}", 10)) - cfg := Config{EnableEmoji: false} - - b.ResetTimer() - for i := 0; i < b.N; i++ { - if _, err := parseBytes(input, cfg, lexIntroSection); err != nil { - b.Fatal(err) - } - } -} - -func BenchmarkParseWithEmoji(b *testing.B) { - start := ` - - ---- -title: "Front Matters" -description: "It really does" ---- - -This is some summary. This is some summary. This is some summary. This is some summary. - - - - -` - input := []byte(start + strings.Repeat("this is not emoji: ", 50) + strings.Repeat("some text ", 70) + strings.Repeat("this is not: ", 50) + strings.Repeat("but this is a :smile: ", 3) + strings.Repeat("some text ", 70)) - cfg := Config{EnableEmoji: true} + cfg := Config{} b.ResetTimer() for i := 0; i < b.N; i++ {