From 08b8c43eeaf9ddebf99f339f479177ed378173de Mon Sep 17 00:00:00 2001 From: Seth Vargo Date: Mon, 16 Nov 2020 12:06:11 -0500 Subject: [PATCH] Add i18n framework Adds a few default translations for demonstration --- cmd/server/assets/codes/issue.html | 6 +- go.mod | 5 +- go.sum | 6 +- internal/i18n/i18n.go | 104 ++++++++++++++++++++++++++++ internal/i18n/i18n_test.go | 81 ++++++++++++++++++++++ internal/i18n/locales/en/default.po | 16 +++++ internal/i18n/locales/es/default.po | 16 +++++ internal/routes/server.go | 11 +++ pkg/controller/middleware/locale.go | 53 ++++++++++++++ pkg/render/render.go | 17 +++++ 10 files changed, 308 insertions(+), 7 deletions(-) create mode 100644 internal/i18n/i18n.go create mode 100644 internal/i18n/i18n_test.go create mode 100644 internal/i18n/locales/en/default.po create mode 100644 internal/i18n/locales/es/default.po create mode 100644 pkg/controller/middleware/locale.go diff --git a/cmd/server/assets/codes/issue.html b/cmd/server/assets/codes/issue.html index 29e36a000..997f33714 100644 --- a/cmd/server/assets/codes/issue.html +++ b/cmd/server/assets/codes/issue.html @@ -35,7 +35,7 @@ {{end}} -

Create verification code

+

{{t $.locale "create-verification-code"}}

Complete the following form to issue a single-use token to verify a patient. Do not submit this form until you are prepared to relay @@ -53,7 +53,7 @@

Create verification code

-
Diagnosis
+
{{t $.locale "diagnosis"}}
{{if $currentRealm.ValidTestType "confirmed"}} @@ -102,7 +102,7 @@

Create verification code

-
Dates
+
{{t $.locale "dates"}}
diff --git a/go.mod b/go.mod index 8c0447472..aeae40f59 100644 --- a/go.mod +++ b/go.mod @@ -42,6 +42,7 @@ require ( github.com/hashicorp/go-retryablehttp v0.6.8 // indirect github.com/jinzhu/gorm v1.9.16 github.com/jinzhu/now v1.1.1 // indirect + github.com/leonelquinteros/gotext v1.4.0 github.com/lib/pq v1.8.0 github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect github.com/microcosm-cc/bluemonday v1.0.4 @@ -67,9 +68,9 @@ require ( golang.org/x/net v0.0.0-20201110031124-69a78807bb2b // indirect golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58 // indirect golang.org/x/sys v0.0.0-20201109165425-215b40eba54c // indirect - golang.org/x/text v0.3.4 // indirect + golang.org/x/text v0.3.4 golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e - golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd + golang.org/x/tools v0.0.0-20201116182000-1d699438d2cf google.golang.org/api v0.35.0 google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a diff --git a/go.sum b/go.sum index 51b59e2db..cafab4f30 100644 --- a/go.sum +++ b/go.sum @@ -921,6 +921,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/leonelquinteros/gotext v1.4.0 h1:2NHPCto5IoMXbrT0bldPrxj0qM5asOCwtb1aUQZ1tys= +github.com/leonelquinteros/gotext v1.4.0/go.mod h1:yZGXREmoGTtBvZHNcc+Yfug49G/2spuF/i/Qlsvz1Us= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -1590,8 +1592,8 @@ golang.org/x/tools v0.0.0-20200915173823-2db8f0ff891c/go.mod h1:z6u4i615ZeAfBE4X golang.org/x/tools v0.0.0-20200918232735-d647fc253266/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/tools v0.0.0-20200923182640-463111b69878/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/tools v0.0.0-20201030143252-cf7a54d06671/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd h1:kJP9fbfkpUoA4y03Nxor8be+YbShcXP16fc7G4nlgpw= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201116182000-1d699438d2cf h1:sDQg8i3k24bqfv1V4MugOhRCHMRzkrHdLJX5QraRSt4= +golang.org/x/tools v0.0.0-20201116182000-1d699438d2cf/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/i18n/i18n.go b/internal/i18n/i18n.go new file mode 100644 index 000000000..42c3b641f --- /dev/null +++ b/internal/i18n/i18n.go @@ -0,0 +1,104 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package i18n defines internationalization and localization. +package i18n + +import ( + "context" + "fmt" + "io/ioutil" + "path/filepath" + + "github.com/leonelquinteros/gotext" + "golang.org/x/text/language" + + "github.com/google/exposure-notifications-verification-server/internal/project" +) + +const ( + // defaultLocale is the default fallback locale when all else fails. + defaultLocale = "en" + + // defaultDomain is the domain to load. + defaultDomain = "default" +) + +// LocaleMap is a map of locale names to their data structure. +type LocaleMap struct { + data map[string]*gotext.Locale + matcher language.Matcher +} + +// Lookup finds the best locale for the given ids. If none exists, the default +// locale is used. +func (l *LocaleMap) Lookup(ids ...string) *gotext.Locale { + for _, id := range ids { + // Convert the id to the "canonical" form. + canonical, err := l.Canonicalize(id) + if err != nil { + continue + } + locale, ok := l.data[canonical] + if !ok { + continue + } + return locale + } + + return l.data[defaultLocale] +} + +// Canonicalize converts the given ID to the expected name. +func (l *LocaleMap) Canonicalize(id string) (string, error) { + desired, _, err := language.ParseAcceptLanguage(id) + if err != nil { + return "", err + } + if tag, _, conf := l.matcher.Match(desired...); conf != language.No { + raw, _, _ := tag.Raw() + return raw.String(), nil + } + return "", fmt.Errorf("unknown language %q", id) +} + +// Load parses and loads the localization files from disk. It builds the locale +// matcher based on the currently available data (organized by folder). +// +// Due to the heavy I/O, callers should cache the resulting value and only call +// Load when data needs to be refreshed. +func Load(ctx context.Context) (*LocaleMap, error) { + localesDir := filepath.Join(project.Root(), "internal", "i18n", "locales") + + entries, err := ioutil.ReadDir(localesDir) + if err != nil { + return nil, fmt.Errorf("failed to load locales: %w", err) + } + + data := make(map[string]*gotext.Locale, len(entries)) + names := make([]language.Tag, 0, len(entries)) + for _, entry := range entries { + name := entry.Name() + names = append(names, language.Make(name)) + + locale := gotext.NewLocale(localesDir, name) + locale.AddDomain(defaultDomain) + data[name] = locale + } + + return &LocaleMap{ + data: data, + matcher: language.NewMatcher(names), + }, nil +} diff --git a/internal/i18n/i18n_test.go b/internal/i18n/i18n_test.go new file mode 100644 index 000000000..b320ddc21 --- /dev/null +++ b/internal/i18n/i18n_test.go @@ -0,0 +1,81 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package i18n + +import ( + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/google/exposure-notifications-verification-server/internal/project" + "github.com/leonelquinteros/gotext" +) + +// TestI18n_matching constructs the superset of all i18n strings and then +// ensures all translation files define said strings. +func TestI18n_matching(t *testing.T) { + t.Parallel() + + var pos []*gotext.Po + localesDir := filepath.Join(project.Root(), "internal", "i18n", "locales") + if err := filepath.Walk(localesDir, func(pth string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() { + return nil + } + + if filepath.Ext(info.Name()) != ".po" { + return nil + } + + po := new(gotext.Po) + po.ParseFile(pth) + pos = append(pos, po) + return nil + }); err != nil { + t.Fatal(err) + } + + // This will almost certainly come back to bite me, but the only way to access + // the actual "list" of translations is to access a private field with + // reflect. Please don't try this at home kids. + translations := make(map[string]struct{}) + translationsByLocale := make(map[string]map[string]struct{}) + for _, po := range pos { + keys := reflect.ValueOf(po).Elem().FieldByName("translations").MapKeys() + for _, v := range keys { + if s := v.String(); s != "" { + translations[s] = struct{}{} + + if translationsByLocale[po.Language] == nil { + translationsByLocale[po.Language] = make(map[string]struct{}) + } + translationsByLocale[po.Language][s] = struct{}{} + } + } + } + + for k := range translations { + for loc, existing := range translationsByLocale { + if _, ok := existing[k]; !ok { + t.Errorf("locale %q is missing translation %q", loc, k) + } + } + } +} diff --git a/internal/i18n/locales/en/default.po b/internal/i18n/locales/en/default.po new file mode 100644 index 000000000..18081ea48 --- /dev/null +++ b/internal/i18n/locales/en/default.po @@ -0,0 +1,16 @@ +msgid "" +msgstr "" +"Language: en\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" + +msgid "create-verification-code" +msgstr "Create verification code" + +msgid "diagnosis" +msgstr "Diagnosis" + +msgid "dates" +msgstr "Dates" diff --git a/internal/i18n/locales/es/default.po b/internal/i18n/locales/es/default.po new file mode 100644 index 000000000..84a9ffef0 --- /dev/null +++ b/internal/i18n/locales/es/default.po @@ -0,0 +1,16 @@ +msgid "" +msgstr "" +"Language: es\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" + +msgid "create-verification-code" +msgstr "Crear código de verificación" + +msgid "diagnosis" +msgstr "Diagnóstico" + +msgid "dates" +msgstr "Fechas" diff --git a/internal/routes/server.go b/internal/routes/server.go index 996790b3b..cc2f3e0e4 100644 --- a/internal/routes/server.go +++ b/internal/routes/server.go @@ -21,6 +21,7 @@ import ( "path/filepath" "github.com/google/exposure-notifications-verification-server/internal/auth" + "github.com/google/exposure-notifications-verification-server/internal/i18n" "github.com/google/exposure-notifications-verification-server/pkg/cache" "github.com/google/exposure-notifications-verification-server/pkg/config" "github.com/google/exposure-notifications-verification-server/pkg/controller" @@ -73,6 +74,16 @@ func Server( populateTemplateVariables := middleware.PopulateTemplateVariables(cfg) r.Use(populateTemplateVariables) + // Load localization + locales, err := i18n.Load(ctx) + if err != nil { + return nil, fmt.Errorf("failed to setup i18n: %w", err) + } + + // Process localization parameters. + processLocale := middleware.ProcessLocale(locales) + r.Use(processLocale) + // Create the renderer h, err := render.New(ctx, cfg.AssetsPath, cfg.DevMode) if err != nil { diff --git a/pkg/controller/middleware/locale.go b/pkg/controller/middleware/locale.go new file mode 100644 index 000000000..b311f4dc4 --- /dev/null +++ b/pkg/controller/middleware/locale.go @@ -0,0 +1,53 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package middleware + +import ( + "net/http" + + "github.com/google/exposure-notifications-verification-server/internal/i18n" + "github.com/google/exposure-notifications-verification-server/pkg/controller" + + "github.com/gorilla/mux" +) + +const ( + HeaderAcceptLanguage = "Accept-Language" + QueryKeyLanguage = "lang" +) + +// ProcessLocale extracts the locale from the various possible locations and +// sets the template translator to the correct language. +// +// This must be called after the template map has been created. +func ProcessLocale(locales *i18n.LocaleMap) mux.MiddlewareFunc { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + // TODO(sethvargo): extract from session/cookie as well + param := r.URL.Query().Get(QueryKeyLanguage) + cookie := "" + header := r.Header.Get(HeaderAcceptLanguage) + + // Find the "best" language from the given parameters. They are in + // priority order. + m := controller.TemplateMapFromContext(ctx) + m["locale"] = locales.Lookup(param, cookie, header) + + next.ServeHTTP(w, r) + }) + } +} diff --git a/pkg/render/render.go b/pkg/render/render.go index 26b7dd3af..d1f1a5fe7 100644 --- a/pkg/render/render.go +++ b/pkg/render/render.go @@ -28,6 +28,7 @@ import ( "github.com/google/exposure-notifications-server/pkg/logging" "github.com/google/exposure-notifications-verification-server/internal/project" + "github.com/leonelquinteros/gotext" "go.uber.org/zap" ) @@ -151,6 +152,21 @@ func selectedIf(v bool) htmltemplate.HTML { return "" } +// translate accepts a message printer (populated by middleware) and prints the +// translated text for the given key. If the printer is nil, an error is +// returned. +func translate(l *gotext.Locale, key string, vars ...interface{}) (string, error) { + if l == nil { + return "", fmt.Errorf("missing locale") + } + + v := l.Get(key, vars...) + if v == "" || v == key { + return "", fmt.Errorf("unknown i18n key %q", key) + } + return v, nil +} + func templateFuncs() htmltemplate.FuncMap { return map[string]interface{}{ "joinStrings": strings.Join, @@ -160,6 +176,7 @@ func templateFuncs() htmltemplate.FuncMap { "toUpper": strings.ToUpper, "safeHTML": safeHTML, "selectedIf": selectedIf, + "t": translate, } }