check-po: check typos of unmatched variable names

Variable names, such as "diff.submodule", "add_cacheinfo", should not
be translated.  Try to find variable names in msgid, and show warning
message for unmatched variable names in msgstr.

Signed-off-by: Jiang Xin <zhiyou.jx@alibaba-inc.com>

cmd/check-po.go

@@ -26,7 +26,11 @@ func (v *checkPoCommand) Command() *cobra.Command {
 	v.cmd.Flags().Bool("core",
 		false,
 		"also check XX.po against "+util.CorePot)
+	v.cmd.Flags().Bool("ignore-typos",
+		false,
+		"do not check typos in .po file")
 	viper.BindPFlag("check-po--core", v.cmd.Flags().Lookup("core"))
+	viper.BindPFlag("check-po--ignore-typos", v.cmd.Flags().Lookup("ignore-typos"))
 
 	return v.cmd
 }

cmd/check.go

@@ -25,17 +25,21 @@ func (v *checkCommand) Command() *cobra.Command {
 	}
 	v.cmd.Flags().Bool("no-gpg",
 		false,
-		"do no verify gpg-signed commit")
+		"do not verify gpg-signed commit")
 	v.cmd.Flags().BoolP("force",
 		"f",
 		false,
 		"run even too many commits")
 	v.cmd.Flags().Bool("core",
 		false,
 		"also check XX.po against "+util.CorePot)
+	v.cmd.Flags().Bool("ignore-typos",
+		false,
+		"do not check typos in .po file")
 	viper.BindPFlag("check--no-gpg", v.cmd.Flags().Lookup("no-gpg"))
 	viper.BindPFlag("check--force", v.cmd.Flags().Lookup("force"))
 	viper.BindPFlag("check--core", v.cmd.Flags().Lookup("core"))
+	viper.BindPFlag("check--ignore-typos", v.cmd.Flags().Lookup("ignore-typos"))
 
 	return v.cmd
 }

go.mod

@@ -3,6 +3,7 @@ module github.com/git-l10n/git-po-helper
 go 1.16
 
 require (
+	github.com/gorilla/i18n v0.0.0-20150820051429-8b358169da46
 	github.com/mattn/go-isatty v0.0.3
 	github.com/qiniu/iconv v1.2.0
 	github.com/sirupsen/logrus v1.8.1

go.sum

@@ -68,6 +68,8 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/i18n v0.0.0-20150820051429-8b358169da46 h1:N+R2A3fGIr5GucoRMu2xpqyQWQlfY31orbofBCdjMz8=
+github.com/gorilla/i18n v0.0.0-20150820051429-8b358169da46/go.mod h1:2Yoiy15Cf7Q3NFwfaJquh7Mk1uGI09ytcD7CUhn8j7s=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
@@ -197,7 +199,6 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 h1:58fnuSXlxZmFdJyvtTFVmVhcMLU6v5fEb/ok4wyqtNU=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=

test/t0031-check-typos-in-po.sh

@@ -0,0 +1,78 @@
+#!/bin/sh
+
+test_description="check typos in po files"
+
+. ./lib/sharness.sh
+
+test_expect_success "setup" '
+	git clone "$PO_HELPER_TEST_REPOSITORY" workdir &&
+	test -f workdir/po/git.pot
+'
+
+test_expect_success "check typos in zh_CN.po" '
+	(
+		cd workdir &&
+		test ! -f po/zh_CN.po &&
+
+		cat >po/zh_CN.po <<-\EOF &&
+		msgid ""
+		msgstr ""
+		"Project-Id-Version: Git\n"
+		"Report-Msgid-Bugs-To: Git Mailing List <git@vger.kernel.org>\n"
+		"POT-Creation-Date: 2021-03-04 22:41+0800\n"
+		"PO-Revision-Date: 2021-03-04 22:41+0800\n"
+		"Last-Translator: Automatically generated\n"
+		"Language-Team: none\n"
+		"Language: zh_CN\n"
+		"MIME-Version: 1.0\n"
+		"Content-Type: text/plain; charset=UTF-8\n"
+		"Content-Transfer-Encoding: 8bit\n"
+		"Plural-Forms: nplurals=2; plural=(n != 1);\n"
+
+		msgid "ignore invalid color %.*s in log.graphColors"
+		msgstr "忽略 log.graphColorss 中无效的颜色 %.*s"
+
+		msgid "check settings of core.gitProxy config variable"
+		msgstr "检查 core.gitProxy 配置变量的设置"
+
+		msgid "check settings of config_variable"
+		msgstr "检查配置变量的设置"
+
+		msgid "check settings of <config_variable>"
+		msgstr "检查 <配置变量> 的设置"
+
+		msgid "check settings of [config_variable]"
+		msgstr "检查 [配置变量] 的设置"
+
+		msgid "checking config.variables (one command)"
+		msgid_plural "checking config.variables (%d commands)"
+		msgstr[0] "检查 配置.变量（一条命令）"
+		msgstr[1] "检查 配置.变量（%d 条命令）"
+		EOF
+
+		git-po-helper check-po  zh_CN >out 2>&1 &&
+		make_user_friendly_and_stable_output <out >actual &&
+		cat >expect <<-\EOF &&
+		[po/zh_CN.po] 6 translated messages.
+		level=warning msg="mismatch variable names in msgstr: config_variable"
+		level=warning msg=">> msgid: check settings of config_variable"
+		level=warning msg=">> msgstr: 检查配置变量的设置"
+		level=warning
+		level=warning msg="mismatch variable names in msgstr: config.variables"
+		level=warning msg=">> msgid: checking config.variables (one command)"
+		level=warning msg=">> msgstr: 检查 配置.变量（一条命令）"
+		level=warning
+		level=warning msg="mismatch variable names in msgstr: config.variables"
+		level=warning msg=">> msgid: checking config.variables (%d commands)"
+		level=warning msg=">> msgstr: 检查 配置.变量（%d 条命令）"
+		level=warning
+		level=warning msg="mismatch variable names in msgstr: log.graphColors"
+		level=warning msg=">> msgid: ignore invalid color %.*s in log.graphColors"
+		level=warning msg=">> msgstr: 忽略 log.graphColorss 中无效的颜色 %.*s"
+		level=warning
+		EOF
+		test_cmp expect actual
+	)
+'
+
+test_done

util/gettext.go

@@ -8,7 +8,10 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
+	"strings"
 
+	"github.com/gorilla/i18n/gettext"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
 )
@@ -21,6 +24,15 @@ type Prompt struct {
 	Silence     bool
 }
 
+var (
+	varNamePattern      = regexp.MustCompile(`[<\[]?\b([a-zA-Z]+\.[a-zA-Z]+|[a-zA-Z]+_[a-zA-Z]+)\b[>\]]?`)
+	varNameExcludeWords = []string{
+		"e.g",
+		"i.e",
+		"example.com",
+	}
+)
+
 func (v *Prompt) String() string {
 	return fmt.Sprintf("[%s]", v.ShortPrompt)
 }
@@ -33,6 +45,90 @@ func (v *Prompt) Width() int {
 	return v.PromptWidth
 }
 
+func checkTyposInMoFile(moFile string) {
+	if viper.GetBool("check-po--ignore-typos") || viper.GetBool("check--ignore-typos") {
+		return
+	}
+
+	f, err := os.Open(moFile)
+	if err != nil {
+		log.Errorf("cannot open %s: %s", moFile, err)
+	}
+	defer f.Close()
+	iter := gettext.ReadMo(f)
+	for {
+		msg, err := iter.Next()
+		if err != nil {
+			if err != io.EOF {
+				log.Errorf("fail to iterator: %s\n", err)
+			}
+			break
+		}
+		if len(msg.StrPlural) == 0 {
+			checkTypos(msg.Id, msg.Str)
+		} else {
+			for i := range msg.StrPlural {
+				if i == 0 {
+					checkTypos(msg.Id, msg.StrPlural[i])
+				} else {
+					checkTypos(msg.IdPlural, msg.StrPlural[i])
+				}
+			}
+		}
+	}
+}
+
+func checkTypos(msgID, msgStr []byte) {
+	if len(msgStr) == 0 {
+		return
+	}
+
+	matchesInID := varNamePattern.FindAllStringSubmatch(string(msgID), -1)
+	if len(matchesInID) == 0 {
+		return
+	}
+	matchesInStr := varNamePattern.FindAllStringSubmatch(string(msgStr), -1)
+	unmatched := []string{}
+	for _, m := range matchesInID {
+		// Ignore exclude words
+		foundExclude := false
+		for _, exclude := range varNameExcludeWords {
+			if m[1] == exclude {
+				foundExclude = true
+				break
+			}
+		}
+		if foundExclude {
+			continue
+		}
+
+		// Ignore "<var_name>" and "[var_name]" in msgid.
+		if len(m[0]) == len(m[1])+2 {
+			continue
+		}
+
+		found := false
+		for _, mStr := range matchesInStr {
+			if m[1] == mStr[1] {
+				found = true
+				break
+			}
+		}
+		if found {
+			continue
+		}
+
+		unmatched = append(unmatched, m[1])
+	}
+	if len(unmatched) > 0 {
+		log.Warnf("mismatch variable names in msgstr: %s",
+			strings.Join(unmatched, ", "))
+		log.Warnf(">> msgid: %s", msgID)
+		log.Warnf(">> msgstr: %s", msgStr)
+		log.Warnln("")
+	}
+}
+
 func runPoChecking(poFile string, prompt Prompt, backCompatible bool) bool {
 	var (
 		msgs            []string
@@ -78,9 +174,18 @@ func runPoChecking(poFile string, prompt Prompt, backCompatible bool) bool {
 	if execProgram == "" {
 		execProgram = "msgfmt"
 	}
+
+	moFile, err := ioutil.TempFile("", "mofile")
+	if err != nil {
+		log.Error(err)
+		return false
+	}
+	defer os.Remove(moFile.Name())
+	moFile.Close()
+
 	cmd := exec.Command(execProgram,
 		"-o",
-		"-",
+		moFile.Name(),
 		"--check",
 		"--statistics",
 		poFile)
@@ -117,6 +222,12 @@ func runPoChecking(poFile string, prompt Prompt, backCompatible bool) bool {
 			fmt.Printf("%-*s %s", prompt.Width(), prompt.String(), line)
 		}
 	}
+
+	// Check typos in mo file.
+	if !backCompatible {
+		checkTyposInMoFile(moFile.Name())
+	}
+
 	return ret
 }