Merge pull request #79 from leancodepl/feat/seed

Add seed command

.gitignore

@@ -7,7 +7,9 @@
 *.dll
 *.so
 *.dylib
+
 poe2arb
+!poe2arb/
 
 # Test binary, built with `go test -c`
 *.test
@@ -24,4 +26,4 @@ go.work
 # Goreleaser
 dist/
 
-.vscode
+.vscode

README.md

@@ -78,6 +78,22 @@ Currently, only an stdin/stdout is supported for the `poe2arb convert` command.
 poe2arb convert io --lang en < Hello_World_English.json > lib/l10n/app_en.arb
 ```
 
+### Seeding POEditor project
+
+**EXPERIMENTAL FEATURE**
+
+If you're setting up a project from some template code, you probably already have some ARB files that need
+to be imported into the POEditor project. Using the POEditor's built-in tool won't give a satisfying result,
+as it will completely ignore placeholders along with their types and other parameters, as well as it won't
+"understand" the plural ICU message format. This is where `poe2arb seed` command comes into place.
+
+`poe2arb seed` command uses the same configuration as the `poe2arb poe`, but **it needs API access token with a write
+access**, to create language in the project if needed, and to upload the translations and terms.
+
+This command is meant only for seeding the project, i.e. setting its first contents. It won't override your existing
+translations and won't delete anything. That said, it should still be run with caution and running this on projects
+with already populated translations is inadvisable.
+
 ## Syntax & supported features
 
 Term name must be a valid Dart field name, additionaly, it must start with a

cmd/convert.go

@@ -3,7 +3,7 @@ package cmd
 import (
 	"os"
 
-	"github.com/leancodepl/poe2arb/converter"
+	"github.com/leancodepl/poe2arb/convert/poe2arb"
 	"github.com/spf13/cobra"
 )
 
@@ -40,7 +40,7 @@ func runConvertIo(cmd *cobra.Command, args []string) error {
 	noTemplate, _ := cmd.Flags().GetBool(noTemplateFlag)
 	termPrefix, _ := cmd.Flags().GetString(termPrefixFlag)
 
-	conv := converter.NewConverter(os.Stdin, &converter.ConverterOptions{
+	conv := poe2arb.NewConverter(os.Stdin, &poe2arb.ConverterOptions{
 		Lang:                      lang,
 		Template:                  !noTemplate,
 		RequireResourceAttributes: true,

cmd/poe.go

@@ -8,7 +8,7 @@ import (
 	"regexp"
 	"strings"
 
-	"github.com/leancodepl/poe2arb/converter"
+	"github.com/leancodepl/poe2arb/convert/poe2arb"
 	"github.com/leancodepl/poe2arb/flutter"
 	"github.com/leancodepl/poe2arb/log"
 	"github.com/leancodepl/poe2arb/poeditor"
@@ -242,7 +242,7 @@ func (c *poeCommand) ExportLanguage(lang poeditor.Language, template bool) error
 
 	convertLogSub := logSub.Info("converting JSON to ARB").Sub()
 
-	conv := converter.NewConverter(resp.Body, &converter.ConverterOptions{
+	conv := poe2arb.NewConverter(resp.Body, &poe2arb.ConverterOptions{
 		Lang:                      lang.Code,
 		Template:                  template,
 		RequireResourceAttributes: c.options.RequireResourceAttributes,

cmd/poe2arb.go

@@ -21,6 +21,7 @@ const loggerKey = ctxKey(1)
 func Execute(logger *log.Logger) {
 	rootCmd.AddCommand(convertCmd)
 	rootCmd.AddCommand(poeCmd)
+	rootCmd.AddCommand(seedCmd)
 	rootCmd.AddCommand(versionCmd)
 
 	ctx := context.WithValue(context.Background(), loggerKey, logger)

cmd/seed.go

@@ -0,0 +1,162 @@
+package cmd
+
+import (
+	"bytes"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/leancodepl/poe2arb/convert/arb2poe"
+	"github.com/leancodepl/poe2arb/poeditor"
+	"github.com/spf13/cobra"
+)
+
+var seedCmd = &cobra.Command{
+	Use:           "seed",
+	Short:         "EXPERIMENTAL! Seeds POEditor with data from ARBs. To be used only on empty projects.",
+	SilenceErrors: true,
+	SilenceUsage:  true,
+	RunE:          runSeed,
+}
+
+func init() {
+	seedCmd.Flags().StringP(projectIDFlag, "p", "", "POEditor project ID")
+	seedCmd.Flags().StringP(tokenFlag, "t", "", "POEditor API token")
+	seedCmd.Flags().StringP(termPrefixFlag, "", "", "POEditor term prefix")
+	seedCmd.Flags().StringP(outputDirFlag, "o", "", `Output directory [default: "."]`)
+	seedCmd.Flags().StringSliceP(overrideLangsFlag, "", []string{}, "Override downloaded languages")
+}
+
+func runSeed(cmd *cobra.Command, args []string) error {
+	log := getLogger(cmd)
+
+	fileLog := log.Info("loading options").Sub()
+
+	sel, err := getOptionsSelector(cmd)
+	if err != nil {
+		fileLog.Error("failed: " + err.Error())
+		return err
+	}
+
+	options, err := sel.SelectOptions()
+	if err != nil {
+		fileLog.Error("failed: " + err.Error())
+		return err
+	}
+
+	fileLog = log.Info("reading ARB files in %s", options.OutputDir).Sub()
+
+	var files []string
+	rawFiles, err := os.ReadDir(options.OutputDir)
+	if err != nil {
+		fileLog.Error("failed: " + err.Error())
+		return err
+	}
+	for _, file := range rawFiles {
+		if file.IsDir() {
+			continue
+		}
+
+		fileName := file.Name()
+		if !strings.HasPrefix(fileName, options.ARBPrefix) || filepath.Ext(fileName) != ".arb" {
+			continue
+		}
+
+		files = append(files, filepath.Join(options.OutputDir, fileName))
+	}
+
+	if len(files) == 0 {
+		fileLog.Error("no ARB files found")
+		return err
+	} else {
+		fileLog.Info("found %d ARB files", len(files))
+	}
+
+	poeClient := poeditor.NewClient(options.Token)
+
+	availableLangs, err := poeClient.GetProjectLanguages(options.ProjectID)
+	if err != nil {
+		log.Error("failed fetching languages: " + err.Error())
+		return err
+	}
+
+	first := true
+	for _, filePath := range files {
+		fileLog = log.Info("seeding %s", filepath.Base(filePath)).Sub()
+		fileLog.Info("converting ARB to JSON")
+
+		file, err := os.Open(filePath)
+		if err != nil {
+			fileLog.Error("failed: " + err.Error())
+			return err
+		}
+
+		converter := arb2poe.NewConverter(file, options.TemplateLocale, options.TermPrefix)
+
+		var b bytes.Buffer
+		lang, err := converter.Convert(&b)
+		if err != nil {
+			if errors.Is(err, arb2poe.NoTermsError) {
+				fileLog.Info("no terms to convert")
+				continue
+			}
+
+			fileLog.Error("failed: " + err.Error())
+			return err
+		}
+
+		if len(options.OverrideLangs) > 0 {
+			langFound := false
+			for _, overridenLang := range options.OverrideLangs {
+				if lang == overridenLang {
+					langFound = true
+					break
+				}
+			}
+
+			if !langFound {
+				fileLog.Info("skipping language %s", lang)
+				continue
+			}
+		}
+
+		availableLangFound := false
+		for _, availableLang := range availableLangs {
+			if lang == availableLang.Code {
+				availableLangFound = true
+				break
+			}
+		}
+
+		if !availableLangFound {
+			langLog := fileLog.Info("adding language %s to project", lang).Sub()
+
+			err = poeClient.AddLanguage(options.ProjectID, lang)
+			if err != nil {
+				langLog.Error("failed: " + err.Error())
+				return err
+			}
+		}
+
+		if !first {
+			fileLog.Info("waiting 30 seconds to avoid rate limiting")
+			time.Sleep(30 * time.Second)
+		}
+
+		uploadLog := fileLog.Info("uploading JSON to POEditor").Sub()
+
+		err = poeClient.Upload(options.ProjectID, lang, &b)
+		if err != nil {
+			uploadLog.Error("failed: " + err.Error())
+			return err
+		} else {
+			fileLog.Success("done")
+		}
+
+		first = false
+	}
+
+	return nil
+}

convert/arb.go

@@ -0,0 +1,29 @@
+// Package convert holds structures common to both directions of conversion.
+package convert
+
+import (
+	orderedmap "github.com/wk8/go-ordered-map/v2"
+)
+
+const LocaleKey = "@@locale"
+
+type ARBMessage struct {
+	Name        string
+	Translation string
+	Attributes  *ARBMessageAttributes
+}
+
+type ARBMessageAttributes struct {
+	Description  string                                          `json:"description,omitempty"`
+	Placeholders *orderedmap.OrderedMap[string, *ARBPlaceholder] `json:"placeholders,omitempty"`
+}
+
+func (a *ARBMessageAttributes) IsEmpty() bool {
+	return a.Description == "" && (a.Placeholders == nil || a.Placeholders.Len() == 0)
+}
+
+type ARBPlaceholder struct {
+	Name   string `json:"-"`
+	Type   string `json:"type,omitempty"`
+	Format string `json:"format,omitempty"`
+}