Bugfix - More Word Results for Webster Source

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/microcosm-cc/bluemonday v1.0.22
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/ogier/pflag v0.0.0-20160129220114-45c278ab3607
+	golang.org/x/text v0.7.0
 )
 
 require (

go.sum

@@ -14,6 +14,8 @@ github.com/ogier/pflag v0.0.0-20160129220114-45c278ab3607 h1:db+rES1EpSjP45xOU3h
 github.com/ogier/pflag v0.0.0-20160129220114-45c278ab3607/go.mod h1:zkFki7tvTa0tafRvTBIZTvzYyAu6kQhPZFnshFFPE+g=
 golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA=
 gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

source/normalize.go

@@ -0,0 +1,33 @@
+package source
+
+import (
+	"strings"
+	"unicode"
+
+	"golang.org/x/text/runes"
+	"golang.org/x/text/transform"
+	"golang.org/x/text/unicode/norm"
+)
+
+// RemoveDiacritics takes a text and returns the same text with any diacritics
+// removed. If there's an issue with cleaning the string, the original text is
+// returned unchanged.
+func RemoveDiacritics(text string) string {
+	transformer := transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), norm.NFC)
+	normalized, _, err := transform.String(transformer, text)
+
+	if err != nil {
+		return text
+	}
+
+	return normalized
+}
+
+// EqualFoldPlain reports whether s and t, interpreted as UTF-8 strings, are
+// equal under Unicode case-folding AFTER first having each string normalized.
+func EqualFoldPlain(s, t string) bool {
+	return strings.EqualFold(
+		RemoveDiacritics(s),
+		RemoveDiacritics(t),
+	)
+}

source/normalize_test.go

@@ -0,0 +1,78 @@
+package source
+
+import "testing"
+
+func TestRemoveDiacritics(t *testing.T) {
+	for testName, testData := range map[string]struct {
+		text string
+		want string
+	}{
+		"empty": {
+			text: "",
+			want: "",
+		},
+		"no marks": {
+			text: "tree",
+			want: "tree",
+		},
+		"single mark": {
+			text: "façade",
+			want: "facade",
+		},
+		"multiple marks": {
+			text: "résumé",
+			want: "resume",
+		},
+	} {
+		t.Run(testName, func(t *testing.T) {
+			if got := RemoveDiacritics(testData.text); got != testData.want {
+				t.Errorf("RemoveDiacritics returned wrong value. Got %#v. Want %#v.", got, testData.want)
+			}
+		})
+	}
+}
+
+func TestEqualFoldPlain(t *testing.T) {
+	for testName, testData := range map[string]struct {
+		s    string
+		t    string
+		want bool
+	}{
+		"empty": {
+			s:    "",
+			t:    "",
+			want: true,
+		},
+		"no marks": {
+			s:    "tree",
+			t:    "tree",
+			want: true,
+		},
+		"single mark": {
+			s:    "façade",
+			t:    "facade",
+			want: true,
+		},
+		"multiple marks": {
+			s:    "résumé",
+			t:    "resume",
+			want: true,
+		},
+		"multiple marks and capitalizations": {
+			s:    "Résumé",
+			t:    "resume",
+			want: true,
+		},
+		"different": {
+			s:    "test",
+			t:    "resume",
+			want: false,
+		},
+	} {
+		t.Run(testName, func(t *testing.T) {
+			if got := EqualFoldPlain(testData.s, testData.t); got != testData.want {
+				t.Errorf("EqualFoldPlain returned wrong value. Got %#v. Want %#v.", got, testData.want)
+			}
+		})
+	}
+}

source/webster/apidata.go

@@ -214,7 +214,7 @@ func (r apiDefinitionResults) toResults() []source.DictionaryResult {
 	for _, apiEntry := range r {
 		headword := cleanHeadword(apiEntry.Hwi.Hw)
 
-		if !strings.EqualFold(headword, mainWord) {
+		if !source.EqualFoldPlain(headword, mainWord) {
 			continue
 		}