From 1b15fcfdd747f910f8de0249c342de08522945b4 Mon Sep 17 00:00:00 2001 From: Albert Nigmatzianov Date: Mon, 20 Mar 2017 12:02:01 +0500 Subject: [PATCH] Add TOML support Fixes #61 Updates https://github.com/spf13/hugo/issues/3200 Updates https://github.com/spf13/hugo/issues/2577 --- README.md | 8 +++- goi18n/merge_command.go | 31 +++++++++++++--- goi18n/merge_command_flat_test.go | 6 +-- goi18n/testdata/en-us.flat.toml | 25 +++++++++++++ goi18n/testdata/input/flat/ar-ar.one.json | 45 ----------------------- goi18n/testdata/input/flat/ar-ar.one.toml | 37 +++++++++++++++++++ goi18n/testdata/input/flat/en-us.one.json | 23 ------------ goi18n/testdata/input/flat/en-us.one.yaml | 16 ++++++++ i18n/bundle/bundle.go | 25 +++++++++++-- i18n/translations_test.go | 4 ++ 10 files changed, 140 insertions(+), 80 deletions(-) create mode 100644 goi18n/testdata/en-us.flat.toml delete mode 100644 goi18n/testdata/input/flat/ar-ar.one.json create mode 100644 goi18n/testdata/input/flat/ar-ar.one.toml delete mode 100644 goi18n/testdata/input/flat/en-us.one.json create mode 100644 goi18n/testdata/input/flat/en-us.one.yaml diff --git a/README.md b/README.md index e6075097..7136dc82 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ go-i18n is a Go [package](#i18n-package) and a [command](#goi18n-command) that h * Supports [pluralized strings](http://cldr.unicode.org/index/cldr-spec/plural-rules) for all 200+ languages in the [Unicode Common Locale Data Repository (CLDR)](http://www.unicode.org/cldr/charts/28/supplemental/language_plural_rules.html). * Code and tests are [automatically generated](https://github.com/nicksnyder/go-i18n/tree/master/i18n/language/codegen) from [CLDR data](http://cldr.unicode.org/index/downloads) * Supports strings with named variables using [text/template](http://golang.org/pkg/text/template/) syntax. -* Translation files are simple JSON or YAML. +* Translation files are simple JSON, TOML or YAML. * [Documented](http://godoc.org/github.com/nicksnyder/go-i18n) and [tested](https://travis-ci.org/nicksnyder/go-i18n)! Package i18n [![GoDoc](http://godoc.org/github.com/nicksnyder/go-i18n?status.svg)](http://godoc.org/github.com/nicksnyder/go-i18n/i18n) @@ -118,6 +118,10 @@ Here is an example of the default file format that go-i18n supports: To use a different file format, write a parser for the format and add the parsed translations using [AddTranslation](https://godoc.org/github.com/nicksnyder/go-i18n/i18n#AddTranslation). +Note that TOML only supports the flat format, which is described below. + +More examples of translation files: [JSON](https://github.com/nicksnyder/go-i18n/tree/master/goi18n/testdata/input), [TOML](https://github.com/nicksnyder/go-i18n/blob/master/goi18n/testdata/input/flat/ar-ar.one.toml), [YAML](https://github.com/nicksnyder/go-i18n/blob/master/goi18n/testdata/input/yaml/en-us.one.yaml). + Flat Format ------------- @@ -166,6 +170,8 @@ and name of substructures (ids) should be always a string. If there is only one key in substructure and it is "other", then it's non-plural translation, else plural. +More examples of flat format translation files can be found in [goi18n/testdata/input/flat](https://github.com/nicksnyder/go-i18n/tree/master/goi18n/testdata/input/flat). + Contributions ------------- diff --git a/goi18n/merge_command.go b/goi18n/merge_command.go index 184343e5..1b9d04fd 100644 --- a/goi18n/merge_command.go +++ b/goi18n/merge_command.go @@ -14,6 +14,7 @@ import ( "github.com/nicksnyder/go-i18n/i18n/bundle" "github.com/nicksnyder/go-i18n/i18n/language" "github.com/nicksnyder/go-i18n/i18n/translation" + toml "github.com/pelletier/go-toml" ) type mergeCommand struct { @@ -36,7 +37,7 @@ func (mc *mergeCommand) execute() error { bundle := bundle.New() for _, tf := range mc.translationFiles { if err := bundle.LoadTranslationFile(tf); err != nil { - return fmt.Errorf("failed to load translation file %s because %s\n", tf, err) + return fmt.Errorf("failed to load translation file %s: %s\n", tf, err) } } @@ -91,7 +92,11 @@ func (mc *mergeCommand) parse(arguments []string) { mc.sourceLanguage = *sourceLanguage mc.outdir = *outdir mc.format = *format - mc.flat = *flat + if *format == "toml" { + mc.flat = true + } else { + mc.flat = *flat + } } func (mc *mergeCommand) SetArgs(args []string) { @@ -110,13 +115,13 @@ func (mc *mergeCommand) writeFile(label string, translations []translation.Trans buf, err := mc.marshal(convert(translations)) if err != nil { - return fmt.Errorf("failed to marshal %s strings to %s because %s", localeID, mc.format, err) + return fmt.Errorf("failed to marshal %s strings to %s: %s", localeID, mc.format, err) } filename := filepath.Join(mc.outdir, fmt.Sprintf("%s.%s.%s", localeID, label, mc.format)) if err := ioutil.WriteFile(filename, buf, 0666); err != nil { - return fmt.Errorf("failed to write %s because %s", filename, err) + return fmt.Errorf("failed to write %s: %s", filename, err) } return nil } @@ -152,12 +157,27 @@ func (mc mergeCommand) marshal(v interface{}) ([]byte, error) { switch mc.format { case "json": return json.MarshalIndent(v, "", " ") + case "toml": + return marshalTOML(v) case "yaml": return yaml.Marshal(v) } return nil, fmt.Errorf("unsupported format: %s\n", mc.format) } +func marshalTOML(v interface{}) ([]byte, error) { + m, ok := v.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("invalid format for marshaling to TOML") + } + tree, err := toml.TreeFromMap(m) + if err != nil { + return nil, err + } + s, err := tree.ToTomlString() + return []byte(s), err +} + func usageMerge() { fmt.Printf(`Merge translation files. @@ -207,11 +227,12 @@ Options: -format format goi18n encodes the output translation files in this format. - Supported formats: json, yaml + Supported formats: json, toml, yaml Default: json -flat goi18n writes the output translation files in flat format. + Usage of '-format toml' automitically sets this flag. Default: true `) diff --git a/goi18n/merge_command_flat_test.go b/goi18n/merge_command_flat_test.go index 614785ee..caa892d2 100644 --- a/goi18n/merge_command_flat_test.go +++ b/goi18n/merge_command_flat_test.go @@ -2,12 +2,12 @@ package main import "testing" -func TestMergeExecuteFlatJSON(t *testing.T) { +func TestMergeExecuteFlat(t *testing.T) { files := []string{ - "testdata/input/flat/en-us.one.json", + "testdata/input/flat/en-us.one.yaml", "testdata/input/flat/en-us.two.json", "testdata/input/flat/fr-fr.json", - "testdata/input/flat/ar-ar.one.json", + "testdata/input/flat/ar-ar.one.toml", "testdata/input/flat/ar-ar.two.json", } testFlatMergeExecute(t, files) diff --git a/goi18n/testdata/en-us.flat.toml b/goi18n/testdata/en-us.flat.toml new file mode 100644 index 00000000..5623a6c2 --- /dev/null +++ b/goi18n/testdata/en-us.flat.toml @@ -0,0 +1,25 @@ +[program_greeting] +other = "Hello world" + +[person_greeting] +other = "Hello {{.Person}}" + +[my_height_in_meters] +one = "I am {{.Count}} meter tall." +other = "I am {{.Count}} meters tall." + +[your_unread_email_count] +one = "You have {{.Count}} unread email." +other = "You have {{.Count}} unread emails." + +[person_unread_email_count] +one = "{{.Person}} has {{.Count}} unread email." +other = "{{.Person}} has {{.Count}} unread emails." + +[person_unread_email_count_timeframe] +one = "{{.Person}} has {{.Count}} unread email in the past {{.Timeframe}}." +other = "{{.Person}} has {{.Count}} unread emails in the past {{.Timeframe}}." + +[d_days] +one = "{{.Count}} day" +other = "{{.Count}} days" diff --git a/goi18n/testdata/input/flat/ar-ar.one.json b/goi18n/testdata/input/flat/ar-ar.one.json deleted file mode 100644 index 6ae8d01f..00000000 --- a/goi18n/testdata/input/flat/ar-ar.one.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "d_days": { - "few": "arabic few translation of d_days", - "many": "arabic many translation of d_days", - "one": "", - "other": "", - "two": "", - "zero": "" - }, - - "person_greeting": { - "other": "arabic translation of person_greeting" - }, - - "person_unread_email_count": { - "few": "arabic few translation of person_unread_email_count", - "many": "arabic many translation of person_unread_email_count", - "one": "arabic one translation of person_unread_email_count", - "other": "", - "two": "", - "zero": "" - }, - - "person_unread_email_count_timeframe": { - "few": "", - "many": "", - "one": "", - "other": "", - "two": "", - "zero": "" - }, - - "program_greeting": { - "other": "" - }, - - "your_unread_email_count": { - "few": "", - "many": "", - "one": "", - "other": "", - "two": "", - "zero": "" - } -} diff --git a/goi18n/testdata/input/flat/ar-ar.one.toml b/goi18n/testdata/input/flat/ar-ar.one.toml new file mode 100644 index 00000000..364a62cf --- /dev/null +++ b/goi18n/testdata/input/flat/ar-ar.one.toml @@ -0,0 +1,37 @@ +[d_days] +few = "arabic few translation of d_days" +many = "arabic many translation of d_days" +one = "" +other = "" +two = "" +zero = "" + +[person_greeting] +other = "arabic translation of person_greeting" + +[person_unread_email_count] +few = "arabic few translation of person_unread_email_count" +many = "arabic many translation of person_unread_email_count" +one = "arabic one translation of person_unread_email_count" +other = "" +two = "" +zero = "" + +[person_unread_email_count_timeframe] +few = "" +many = "" +one = "" +other = "" +two = "" +zero = "" + +[program_greeting] +other = "" + +[your_unread_email_count] +few = "" +many = "" +one = "" +other = "" +two = "" +zero = "" diff --git a/goi18n/testdata/input/flat/en-us.one.json b/goi18n/testdata/input/flat/en-us.one.json deleted file mode 100644 index 0d93a893..00000000 --- a/goi18n/testdata/input/flat/en-us.one.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "program_greeting": { - "other": "Hello world" - }, - - "your_unread_email_count": { - "one": "You have {{.Count}} unread email.", - "other": "You have {{.Count}} unread emails." - }, - - "my_height_in_meters": { - "one": "I am {{.Count}} meter tall.", - "other": "I am {{.Count}} meters tall." - }, - - "person_unread_email_count_timeframe": { - "other": "{{.Person}} has {{.Count}} unread email in the past {{.Timeframe}}." - }, - - "d_days": { - "other": "this should get overwritten" - } -} diff --git a/goi18n/testdata/input/flat/en-us.one.yaml b/goi18n/testdata/input/flat/en-us.one.yaml new file mode 100644 index 00000000..02ae0011 --- /dev/null +++ b/goi18n/testdata/input/flat/en-us.one.yaml @@ -0,0 +1,16 @@ +program_greeting: + other: "Hello world" + +your_unread_email_count: + one: "You have {{.Count}} unread email." + other: "You have {{.Count}} unread emails." + +my_height_in_meters: + one: "I am {{.Count}} meter tall." + other: "I am {{.Count}} meters tall." + +person_unread_email_count_timeframe: + other: "{{.Person}} has {{.Count}} unread email in the past {{.Timeframe}}." + +d_days: + other: "this should get overwritten" diff --git a/i18n/bundle/bundle.go b/i18n/bundle/bundle.go index e3bc744c..2ad1d7cc 100644 --- a/i18n/bundle/bundle.go +++ b/i18n/bundle/bundle.go @@ -2,6 +2,7 @@ package bundle import ( + "bytes" "encoding/json" "fmt" "io/ioutil" @@ -11,6 +12,7 @@ import ( "github.com/nicksnyder/go-i18n/i18n/language" "github.com/nicksnyder/go-i18n/i18n/translation" + toml "github.com/pelletier/go-toml" "gopkg.in/yaml.v2" ) @@ -78,12 +80,29 @@ func (b *Bundle) ParseTranslationFileBytes(filename string, buf []byte) error { } func parseTranslations(filename string, buf []byte) ([]translation.Translation, error) { - if buf == nil || len(buf) == 0 { + if len(buf) == 0 { return []translation.Translation{}, nil } ext := filepath.Ext(filename) + // `github.com/pelletier/go-toml` has an Unmarshal function, + // that can't unmarshal to maps, so we should parse TOML format separately. + if ext == ".toml" { + tree, err := toml.LoadReader(bytes.NewReader(buf)) + if err != nil { + return nil, err + } + + m := make(map[string]map[string]interface{}) + for k, v := range tree.ToMap() { + m[k] = v.(map[string]interface{}) + } + + return parseFlatFormat(m) + } + + // Then parse other formats. if isStandardFormat(ext, buf) { var standardFormat []map[string]interface{} if err := unmarshal(ext, buf, &standardFormat); err != nil { @@ -115,9 +134,9 @@ func unmarshal(ext string, buf []byte, out interface{}) error { return json.Unmarshal(buf, out) case ".yaml": return yaml.Unmarshal(buf, out) - default: - return fmt.Errorf("unsupported file extension %v", ext) } + + return fmt.Errorf("unsupported file extension %v", ext) } func parseStandardFormat(data []map[string]interface{}) ([]translation.Translation, error) { diff --git a/i18n/translations_test.go b/i18n/translations_test.go index 70c369b8..86c58083 100644 --- a/i18n/translations_test.go +++ b/i18n/translations_test.go @@ -83,3 +83,7 @@ func TestJSONFlatParse(t *testing.T) { func TestYAMLFlatParse(t *testing.T) { testFile(t, "../goi18n/testdata/en-us.flat.yaml") } + +func TestTOMLFlatParse(t *testing.T) { + testFile(t, "../goi18n/testdata/en-us.flat.toml") +}