From 82778fcfe843efbc27c7b5596b85c129c938133f Mon Sep 17 00:00:00 2001 From: maxprilutskiy Date: Wed, 1 May 2024 16:31:43 +0200 Subject: [PATCH 01/24] feat(cli): add replexica init command --- packages/cli/src/index.ts | 4 ++- packages/cli/src/init.ts | 22 +++++++++++++++ packages/cli/src/services/config.ts | 42 +++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 packages/cli/src/init.ts create mode 100644 packages/cli/src/services/config.ts diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index a41d68629..9863726a6 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -2,6 +2,7 @@ import { Command } from 'commander'; import i18nCmd from './i18n.js'; import authCmd from './auth.js'; +import initCmd from './init.js'; import localizeCmd from './localize/index.js'; export default new Command() @@ -10,5 +11,6 @@ export default new Command() .helpOption('-h, --help', 'Show help') .addCommand(i18nCmd) .addCommand(authCmd) - .addCommand(localizeCmd) + .addCommand(initCmd) + .addCommand(localizeCmd) // legacy .parse(process.argv); diff --git a/packages/cli/src/init.ts b/packages/cli/src/init.ts new file mode 100644 index 000000000..ed0cb93f0 --- /dev/null +++ b/packages/cli/src/init.ts @@ -0,0 +1,22 @@ +import { Command } from "commander"; +import Ora from 'ora'; +import { createEmptyConfig, loadConfig, saveConfig } from "./services/config.js"; + +export default new Command() + .command("init") + .description("Initialize Replexica project") + .helpOption("-h, --help", "Show help") + .action(async (options) => { + const spinner = Ora().start('Initializing Replexica project'); + + let config = await loadConfig(); + if (config) { + spinner.fail('Replexica project already initialized'); + return process.exit(1); + } + + config = await createEmptyConfig(); + await saveConfig(config); + + spinner.succeed('Replexica project initialized'); + }); diff --git a/packages/cli/src/services/config.ts b/packages/cli/src/services/config.ts new file mode 100644 index 000000000..0181113c6 --- /dev/null +++ b/packages/cli/src/services/config.ts @@ -0,0 +1,42 @@ +import Z from 'zod'; +import fs from 'fs'; +import path from 'path'; + +const configFile = "i18n.json"; +const configFilePath = path.join(process.cwd(), configFile); + +const configFileSchema = Z.object({ + version: Z.literal(1), + debug: Z.boolean().default(false), + locale: Z.object({ + source: Z.string(), + targets: Z.array(Z.string()), + }), +}); + +export async function loadConfig(): Promise | null> { + const configFileExists = await fs.existsSync(configFilePath); + if (!configFileExists) { return null; } + + const fileContents = fs.readFileSync(configFilePath, "utf8"); + const rawConfig = JSON.parse(fileContents); + const config = configFileSchema.parse(rawConfig); + + return config; +} + +export async function createEmptyConfig(): Promise> { + return { + version: 1, + debug: false, + locale: { + source: 'en', + targets: ['es'], + }, + }; +} + +export async function saveConfig(config: Z.infer) { + const serialized = JSON.stringify(config, null, 2); + fs.writeFileSync(configFilePath, serialized); +} From c57d9203f4603342d0cb1f628346f09a22ff8fd8 Mon Sep 17 00:00:00 2001 From: maxprilutskiy Date: Wed, 1 May 2024 16:33:20 +0200 Subject: [PATCH 02/24] chore: moved classic demo dir to /demo --- demo/classic/i18n.json | 10 +++++++ .../cli/demo => demo/classic}/json/en.json | 0 .../cli/demo => demo/classic}/json/es.json | 0 .../cli/demo => demo/classic}/json/fr.json | 0 .../cli/demo => demo/classic}/markdown/en.md | 0 .../cli/demo => demo/classic}/markdown/es.md | 0 .../cli/demo => demo/classic}/markdown/fr.md | 0 demo/classic/package.json | 11 +++++++ .../classic}/xcode/Localizable.xcstrings | 0 .../classic}/yaml-root-key/en.yml | 0 .../classic}/yaml-root-key/es.yml | 0 .../classic}/yaml-root-key/fr.yml | 0 .../cli/demo => demo/classic}/yaml/en.yml | 0 .../cli/demo => demo/classic}/yaml/es.yml | 0 .../cli/demo => demo/classic}/yaml/fr.yml | 0 demo/node/CHANGELOG.md | 29 ------------------- demo/node/index.cjs | 9 ------ demo/node/index.js | 9 ------ demo/node/index.mjs | 9 ------ demo/node/package.json | 17 ----------- 20 files changed, 21 insertions(+), 73 deletions(-) create mode 100644 demo/classic/i18n.json rename {packages/cli/demo => demo/classic}/json/en.json (100%) rename {packages/cli/demo => demo/classic}/json/es.json (100%) rename {packages/cli/demo => demo/classic}/json/fr.json (100%) rename {packages/cli/demo => demo/classic}/markdown/en.md (100%) rename {packages/cli/demo => demo/classic}/markdown/es.md (100%) rename {packages/cli/demo => demo/classic}/markdown/fr.md (100%) create mode 100644 demo/classic/package.json rename {packages/cli/demo => demo/classic}/xcode/Localizable.xcstrings (100%) rename {packages/cli/demo => demo/classic}/yaml-root-key/en.yml (100%) rename {packages/cli/demo => demo/classic}/yaml-root-key/es.yml (100%) rename {packages/cli/demo => demo/classic}/yaml-root-key/fr.yml (100%) rename {packages/cli/demo => demo/classic}/yaml/en.yml (100%) rename {packages/cli/demo => demo/classic}/yaml/es.yml (100%) rename {packages/cli/demo => demo/classic}/yaml/fr.yml (100%) delete mode 100644 demo/node/CHANGELOG.md delete mode 100644 demo/node/index.cjs delete mode 100644 demo/node/index.js delete mode 100644 demo/node/index.mjs delete mode 100644 demo/node/package.json diff --git a/demo/classic/i18n.json b/demo/classic/i18n.json new file mode 100644 index 000000000..5ded2b5dd --- /dev/null +++ b/demo/classic/i18n.json @@ -0,0 +1,10 @@ +{ + "version": 1, + "debug": false, + "locale": { + "source": "en", + "targets": [ + "es" + ] + } +} \ No newline at end of file diff --git a/packages/cli/demo/json/en.json b/demo/classic/json/en.json similarity index 100% rename from packages/cli/demo/json/en.json rename to demo/classic/json/en.json diff --git a/packages/cli/demo/json/es.json b/demo/classic/json/es.json similarity index 100% rename from packages/cli/demo/json/es.json rename to demo/classic/json/es.json diff --git a/packages/cli/demo/json/fr.json b/demo/classic/json/fr.json similarity index 100% rename from packages/cli/demo/json/fr.json rename to demo/classic/json/fr.json diff --git a/packages/cli/demo/markdown/en.md b/demo/classic/markdown/en.md similarity index 100% rename from packages/cli/demo/markdown/en.md rename to demo/classic/markdown/en.md diff --git a/packages/cli/demo/markdown/es.md b/demo/classic/markdown/es.md similarity index 100% rename from packages/cli/demo/markdown/es.md rename to demo/classic/markdown/es.md diff --git a/packages/cli/demo/markdown/fr.md b/demo/classic/markdown/fr.md similarity index 100% rename from packages/cli/demo/markdown/fr.md rename to demo/classic/markdown/fr.md diff --git a/demo/classic/package.json b/demo/classic/package.json new file mode 100644 index 000000000..119ace7d5 --- /dev/null +++ b/demo/classic/package.json @@ -0,0 +1,11 @@ +{ + "name": "@replexica/demo/classic", + "private": true, + "version": "0.0.0", + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "replexica": "workspace:*" + } +} diff --git a/packages/cli/demo/xcode/Localizable.xcstrings b/demo/classic/xcode/Localizable.xcstrings similarity index 100% rename from packages/cli/demo/xcode/Localizable.xcstrings rename to demo/classic/xcode/Localizable.xcstrings diff --git a/packages/cli/demo/yaml-root-key/en.yml b/demo/classic/yaml-root-key/en.yml similarity index 100% rename from packages/cli/demo/yaml-root-key/en.yml rename to demo/classic/yaml-root-key/en.yml diff --git a/packages/cli/demo/yaml-root-key/es.yml b/demo/classic/yaml-root-key/es.yml similarity index 100% rename from packages/cli/demo/yaml-root-key/es.yml rename to demo/classic/yaml-root-key/es.yml diff --git a/packages/cli/demo/yaml-root-key/fr.yml b/demo/classic/yaml-root-key/fr.yml similarity index 100% rename from packages/cli/demo/yaml-root-key/fr.yml rename to demo/classic/yaml-root-key/fr.yml diff --git a/packages/cli/demo/yaml/en.yml b/demo/classic/yaml/en.yml similarity index 100% rename from packages/cli/demo/yaml/en.yml rename to demo/classic/yaml/en.yml diff --git a/packages/cli/demo/yaml/es.yml b/demo/classic/yaml/es.yml similarity index 100% rename from packages/cli/demo/yaml/es.yml rename to demo/classic/yaml/es.yml diff --git a/packages/cli/demo/yaml/fr.yml b/demo/classic/yaml/fr.yml similarity index 100% rename from packages/cli/demo/yaml/fr.yml rename to demo/classic/yaml/fr.yml diff --git a/demo/node/CHANGELOG.md b/demo/node/CHANGELOG.md deleted file mode 100644 index 53fb160e2..000000000 --- a/demo/node/CHANGELOG.md +++ /dev/null @@ -1,29 +0,0 @@ -# @replexica/demo/node - -## 0.0.4 - -### Patch Changes - -- Updated dependencies [[`e324c0f`](https://github.com/replexica/replexica/commit/e324c0f22224e102bda6b516014fae82f7bfca32), [`9c3dc98`](https://github.com/replexica/replexica/commit/9c3dc9896b96d755a4d7de8c81a12638c456653c)]: - - @replexica/compiler@0.2.1 - -## 0.0.3 - -### Patch Changes - -- Updated dependencies [[`392bb1c`](https://github.com/replexica/replexica/commit/392bb1cbf35a7b8b11f14788497bd8d36de12808)]: - - @replexica/compiler@0.2.0 - -## 0.0.2 - -### Patch Changes - -- Updated dependencies [[`eca4595`](https://github.com/replexica/replexica/commit/eca45954360f59d57e26ff8dea5841c25bf2f1b7)]: - - @replexica/compiler@0.1.1 - -## 0.0.1 - -### Patch Changes - -- Updated dependencies [[`0c92f9d`](https://github.com/replexica/replexica/commit/0c92f9d3f63f0a6dd0254c90523958ada6348fb6)]: - - @replexica/compiler@0.1.0 diff --git a/demo/node/index.cjs b/demo/node/index.cjs deleted file mode 100644 index a9f700f18..000000000 --- a/demo/node/index.cjs +++ /dev/null @@ -1,9 +0,0 @@ -const compiler = require('@replexica/compiler'); - -console.log(compiler); - -console.log( - compiler.compile( - compiler.code, - ), -); \ No newline at end of file diff --git a/demo/node/index.js b/demo/node/index.js deleted file mode 100644 index a9f700f18..000000000 --- a/demo/node/index.js +++ /dev/null @@ -1,9 +0,0 @@ -const compiler = require('@replexica/compiler'); - -console.log(compiler); - -console.log( - compiler.compile( - compiler.code, - ), -); \ No newline at end of file diff --git a/demo/node/index.mjs b/demo/node/index.mjs deleted file mode 100644 index 367d2ca0f..000000000 --- a/demo/node/index.mjs +++ /dev/null @@ -1,9 +0,0 @@ -import compiler from '@replexica/compiler'; - -console.log(compiler); - -console.log( - compiler.compile( - compiler.code, - ), -); \ No newline at end of file diff --git a/demo/node/package.json b/demo/node/package.json deleted file mode 100644 index 8f31e8615..000000000 --- a/demo/node/package.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "name": "@replexica/demo/node", - "version": "0.0.4", - "private": true, - "description": "", - "scripts": { - "mjs": "node index.mjs", - "cjs": "node index.cjs", - "js": "node index.js" - }, - "dependencies": { - "@replexica/compiler": "workspace:*" - }, - "keywords": [], - "author": "", - "license": "ISC" -} From 068a6f0b021352fac191e4cbfa1636505136c60e Mon Sep 17 00:00:00 2001 From: maxprilutskiy Date: Wed, 1 May 2024 16:33:42 +0200 Subject: [PATCH 03/24] chore: upd lockfile --- pnpm-lock.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 020b95129..dcb2c3be6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -28,6 +28,12 @@ importers: specifier: latest version: 1.13.0 + demo/classic: + dependencies: + replexica: + specifier: workspace:* + version: link:../../packages/cli + demo/next-app: dependencies: '@replexica/react': @@ -172,12 +178,6 @@ importers: specifier: ^5 version: 5.4.5 - demo/node: - dependencies: - '@replexica/compiler': - specifier: workspace:* - version: link:../../packages/compiler - packages/cli: dependencies: '@inquirer/prompts': From 5bb729245737c25f78f7a43b5a421c5e0c9bb53f Mon Sep 17 00:00:00 2001 From: maxprilutskiy Date: Wed, 1 May 2024 16:54:51 +0200 Subject: [PATCH 04/24] feat(cli): add `projects` to i18n.json --- demo/classic/i18n.json | 24 +++++++++++++++++++++++- packages/cli/src/services/config.ts | 16 ++++++++++++---- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/demo/classic/i18n.json b/demo/classic/i18n.json index 5ded2b5dd..af653f9d8 100644 --- a/demo/classic/i18n.json +++ b/demo/classic/i18n.json @@ -6,5 +6,27 @@ "targets": [ "es" ] - } + }, + "projects": [ + { + "type": "json", + "dictionary": "json/[lang].json" + }, + { + "type": "xcode", + "dictionary": "xcode/Localizable.xcstrings" + }, + { + "type": "yaml", + "dictionary": "yaml/[lang].yml" + }, + { + "type": "yaml-root-key", + "dictionary": "yaml-root-key/[lang].yml" + }, + { + "type": "markdown", + "dictionary": "markdown/[lang].md" + } + ] } \ No newline at end of file diff --git a/packages/cli/src/services/config.ts b/packages/cli/src/services/config.ts index 0181113c6..45d9de443 100644 --- a/packages/cli/src/services/config.ts +++ b/packages/cli/src/services/config.ts @@ -5,13 +5,21 @@ import path from 'path'; const configFile = "i18n.json"; const configFilePath = path.join(process.cwd(), configFile); +const localeSchema = Z.object({ + source: Z.string(), + targets: Z.array(Z.string()), +}); + +const projectSchema = Z.object({ + type: Z.enum(["json", "markdown", "yaml", "xcode", "yaml-root-key"]), + dictionary: Z.string(), +}); + const configFileSchema = Z.object({ version: Z.literal(1), debug: Z.boolean().default(false), - locale: Z.object({ - source: Z.string(), - targets: Z.array(Z.string()), - }), + locale: localeSchema, + projects: Z.array(projectSchema).default([]).optional(), }); export async function loadConfig(): Promise | null> { From ab0906639b8bf57f37f2bfc1b4a0a15d25edf069 Mon Sep 17 00:00:00 2001 From: maxprilutskiy Date: Wed, 1 May 2024 16:58:36 +0200 Subject: [PATCH 05/24] feat(spec): added `@replexica/spec` package --- packages/spec/build.config.ts | 5 +++++ packages/spec/package.json | 29 +++++++++++++++++++++++++++++ packages/spec/src/index.spec.ts | 7 +++++++ packages/spec/src/index.ts | 1 + packages/spec/tsconfig.base.json | 17 +++++++++++++++++ packages/spec/tsconfig.json | 9 +++++++++ packages/spec/tsconfig.test.json | 9 +++++++++ pnpm-lock.yaml | 16 ++++++++++++++++ 8 files changed, 93 insertions(+) create mode 100644 packages/spec/build.config.ts create mode 100644 packages/spec/package.json create mode 100644 packages/spec/src/index.spec.ts create mode 100644 packages/spec/src/index.ts create mode 100644 packages/spec/tsconfig.base.json create mode 100644 packages/spec/tsconfig.json create mode 100644 packages/spec/tsconfig.test.json diff --git a/packages/spec/build.config.ts b/packages/spec/build.config.ts new file mode 100644 index 000000000..26684c6d9 --- /dev/null +++ b/packages/spec/build.config.ts @@ -0,0 +1,5 @@ +import { defineBuildConfig } from "unbuild"; + +export default defineBuildConfig({ + outDir: "build", +}); diff --git a/packages/spec/package.json b/packages/spec/package.json new file mode 100644 index 000000000..01388f3ee --- /dev/null +++ b/packages/spec/package.json @@ -0,0 +1,29 @@ +{ + "name": "@replexica/spec", + "version": "0.0.0", + "description": "Replexica open specification", + "private": false, + "type": "module", + "main": "build/index.cjs", + "types": "build/index.d.ts", + "module": "build/index.mjs", + "files": [ + "build" + ], + "scripts": { + "dev": "unbuild --stub && tsc -w --noEmit", + "build": "unbuild", + "test": "vitest run" + }, + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "typescript": "^5.4.5", + "vitest": "^1.4.0", + "zod": "^3.23.0" + }, + "devDependencies": { + "unbuild": "^2.0.0" + } +} diff --git a/packages/spec/src/index.spec.ts b/packages/spec/src/index.spec.ts new file mode 100644 index 000000000..48b99ef9f --- /dev/null +++ b/packages/spec/src/index.spec.ts @@ -0,0 +1,7 @@ +import { describe, it, expect } from 'vitest'; + +describe('Test suite', () => { + it('should pass', () => { + expect(1).toBe(1); + }); +}); \ No newline at end of file diff --git a/packages/spec/src/index.ts b/packages/spec/src/index.ts new file mode 100644 index 000000000..cb0ff5c3b --- /dev/null +++ b/packages/spec/src/index.ts @@ -0,0 +1 @@ +export {}; diff --git a/packages/spec/tsconfig.base.json b/packages/spec/tsconfig.base.json new file mode 100644 index 000000000..5a61c906e --- /dev/null +++ b/packages/spec/tsconfig.base.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "rootDir": "src", + "outDir": "build", + "strict": true, + "esModuleInterop": true, + "allowSyntheticDefaultImports": true, + "skipLibCheck": true, + "jsx": "react-jsx", + "moduleResolution": "Node", + "module": "ESNext", + "target": "ESNext" + } +} diff --git a/packages/spec/tsconfig.json b/packages/spec/tsconfig.json new file mode 100644 index 000000000..d9e677ff8 --- /dev/null +++ b/packages/spec/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "./tsconfig.base.json", + "include": [ + "src/**/*.ts" + ], + "exclude": [ + "src/**/*.spec.ts" + ] +} diff --git a/packages/spec/tsconfig.test.json b/packages/spec/tsconfig.test.json new file mode 100644 index 000000000..c4c382f0a --- /dev/null +++ b/packages/spec/tsconfig.test.json @@ -0,0 +1,9 @@ +{ + "extends": "./tsconfig.base.json", + "include": [ + "src/**/*.ts" + ], + "compilerOptions": { + "noEmit": true + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index dcb2c3be6..82d3e451e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -307,6 +307,22 @@ importers: specifier: ^14 version: 14.1.4(react-dom@18.2.0)(react@18.2.0) + packages/spec: + dependencies: + typescript: + specifier: ^5.4.5 + version: 5.4.5 + vitest: + specifier: ^1.4.0 + version: 1.4.0(@types/node@20.12.5) + zod: + specifier: ^3.23.0 + version: 3.23.0 + devDependencies: + unbuild: + specifier: ^2.0.0 + version: 2.0.0(typescript@5.4.5) + packages: /@alloc/quick-lru@5.2.0: From 1ffb1b4bb5d0d5022817fb0962c48074275f10a7 Mon Sep 17 00:00:00 2001 From: maxprilutskiy Date: Wed, 1 May 2024 17:17:31 +0200 Subject: [PATCH 06/24] chore(spec): migrate compiler + cli to use spec --- packages/cli/package.json | 1 + packages/cli/src/localize/config/schema.ts | 38 ++++++++-------------- packages/cli/src/services/config.ts | 9 ++--- packages/compiler/package.json | 1 + packages/compiler/src/options.ts | 6 ++-- packages/spec/src/formats.ts | 5 +++ packages/spec/src/index.ts | 3 +- packages/spec/src/locales.ts | 11 +++++++ pnpm-lock.yaml | 6 ++++ 9 files changed, 47 insertions(+), 33 deletions(-) create mode 100644 packages/spec/src/formats.ts create mode 100644 packages/spec/src/locales.ts diff --git a/packages/cli/package.json b/packages/cli/package.json index c9489a199..5c7125caf 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -23,6 +23,7 @@ "author": "", "license": "ISC", "dependencies": { + "@replexica/spec": "workspace:*", "@inquirer/prompts": "^4.3.1", "@paralleldrive/cuid2": "^2.2.2", "commander": "^12.0.0", diff --git a/packages/cli/src/localize/config/schema.ts b/packages/cli/src/localize/config/schema.ts index d133fdbd0..760fa987e 100644 --- a/packages/cli/src/localize/config/schema.ts +++ b/packages/cli/src/localize/config/schema.ts @@ -1,33 +1,21 @@ -import { z } from 'zod'; +import Z from 'zod'; +import { sourceLocaleSchema, targetLocaleSchema, projectTypeSchema } from '@replexica/spec'; -const supportedLanguages = z.enum(['en', 'es', 'ca', 'fr', 'it', 'de', 'ja', 'ko', 'zh-CN', 'ru']); - -const supportedProjectTypes = z.enum(['json', 'xcode', 'yaml', 'yaml-root-key', 'markdown']); - -const languageSchema = z.object({ - source: supportedLanguages, - target: z.array(supportedLanguages).transform((val) => { - if (val.length === 0) { - throw new Error('target languages must not be empty'); - } - const uniqueTargetLangs = [...new Set(val)]; - if (uniqueTargetLangs.length !== val.length) { - throw new Error('target languages must be unique'); - } - return uniqueTargetLangs; - }), +const languageSchema = Z.object({ + source: sourceLocaleSchema, + target: Z.array(targetLocaleSchema), }); -const projectSchema = z.object({ - name: z.string(), - type: supportedProjectTypes.optional().default('json'), - dictionary: z.string(), +const projectSchema = Z.object({ + name: Z.string(), + type: projectTypeSchema.optional().default('json'), + dictionary: Z.string(), }); -export const configSchema = z.object({ - version: z.literal(1), +export const configSchema = Z.object({ + version: Z.literal(1), languages: languageSchema, - projects: z.array(projectSchema), + projects: Z.array(projectSchema), }); -export type ConfigSchema = z.infer; +export type ConfigSchema = Z.infer; diff --git a/packages/cli/src/services/config.ts b/packages/cli/src/services/config.ts index 45d9de443..db53e8c74 100644 --- a/packages/cli/src/services/config.ts +++ b/packages/cli/src/services/config.ts @@ -1,17 +1,18 @@ import Z from 'zod'; import fs from 'fs'; import path from 'path'; +import { projectTypeSchema, sourceLocaleSchema, targetLocaleSchema } from '@replexica/spec'; const configFile = "i18n.json"; const configFilePath = path.join(process.cwd(), configFile); const localeSchema = Z.object({ - source: Z.string(), - targets: Z.array(Z.string()), + source: sourceLocaleSchema, + targets: Z.array(targetLocaleSchema), }); const projectSchema = Z.object({ - type: Z.enum(["json", "markdown", "yaml", "xcode", "yaml-root-key"]), + type: projectTypeSchema, dictionary: Z.string(), }); @@ -26,7 +27,7 @@ export async function loadConfig(): Promise | n const configFileExists = await fs.existsSync(configFilePath); if (!configFileExists) { return null; } - const fileContents = fs.readFileSync(configFilePath, "utf8"); + const fileContents = fs.readFileSync(configFilePath, "utf8"); const rawConfig = JSON.parse(fileContents); const config = configFileSchema.parse(rawConfig); diff --git a/packages/compiler/package.json b/packages/compiler/package.json index 9acffa5d3..0738281de 100644 --- a/packages/compiler/package.json +++ b/packages/compiler/package.json @@ -19,6 +19,7 @@ "author": "", "license": "ISC", "dependencies": { + "@replexica/spec": "workspace:*", "@babel/core": "^7.24.4", "@babel/generator": "^7.24.4", "@babel/parser": "^7.24.4", diff --git a/packages/compiler/src/options.ts b/packages/compiler/src/options.ts index 9e623b510..0a4906734 100644 --- a/packages/compiler/src/options.ts +++ b/packages/compiler/src/options.ts @@ -1,10 +1,10 @@ import Z from 'zod'; +import { sourceLocaleSchema, targetLocaleSchema } from '@replexica/spec'; -const supportedLocale = Z.enum(['en', 'es']); const optionsSchema = Z.object({ locale: Z.object({ - source: supportedLocale, - targets: Z.array(supportedLocale), + source: sourceLocaleSchema, + targets: Z.array(targetLocaleSchema), }), rsc: Z.boolean().optional().default(true), debug: Z.boolean().optional().default(false), diff --git a/packages/spec/src/formats.ts b/packages/spec/src/formats.ts new file mode 100644 index 000000000..c2b491ee7 --- /dev/null +++ b/packages/spec/src/formats.ts @@ -0,0 +1,5 @@ +import Z from 'zod'; + +export const projectTypes = ['json', 'markdown', 'yaml', 'xcode', 'yaml-root-key'] as const; + +export const projectTypeSchema = Z.enum(projectTypes); \ No newline at end of file diff --git a/packages/spec/src/index.ts b/packages/spec/src/index.ts index cb0ff5c3b..6c64b3d7d 100644 --- a/packages/spec/src/index.ts +++ b/packages/spec/src/index.ts @@ -1 +1,2 @@ -export {}; +export * from './locales'; +export * from './formats'; diff --git a/packages/spec/src/locales.ts b/packages/spec/src/locales.ts new file mode 100644 index 000000000..c7ec08005 --- /dev/null +++ b/packages/spec/src/locales.ts @@ -0,0 +1,11 @@ +import Z from 'zod'; + +// Source +export const sourceLocales = ['en'] as const; + +export const sourceLocaleSchema = Z.enum(sourceLocales); + +// Target +export const targetLocales = ['es'] as const; + +export const targetLocaleSchema = Z.enum(targetLocales); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 82d3e451e..a019b688a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -186,6 +186,9 @@ importers: '@paralleldrive/cuid2': specifier: ^2.2.2 version: 2.2.2 + '@replexica/spec': + specifier: workspace:* + version: link:../spec commander: specifier: ^12.0.0 version: 12.0.0 @@ -256,6 +259,9 @@ importers: '@babel/types': specifier: ^7.24.0 version: 7.24.0 + '@replexica/spec': + specifier: workspace:* + version: link:../spec ignore-walk: specifier: ^6.0.4 version: 6.0.4 From 76112c2deba9e06cffa404339ab91190b8b67493 Mon Sep 17 00:00:00 2001 From: maxprilutskiy Date: Wed, 1 May 2024 17:55:49 +0200 Subject: [PATCH 07/24] refactor(spec): refactored i18n.json format --- demo/classic/i18n.json | 14 +++++++------- packages/cli/src/services/config.ts | 11 ++++++----- packages/spec/src/formats.ts | 4 ++-- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/demo/classic/i18n.json b/demo/classic/i18n.json index af653f9d8..7e4f4a29c 100644 --- a/demo/classic/i18n.json +++ b/demo/classic/i18n.json @@ -7,26 +7,26 @@ "es" ] }, - "projects": [ + "content": [ { "type": "json", - "dictionary": "json/[lang].json" + "path": "json/[lang].json" }, { "type": "xcode", - "dictionary": "xcode/Localizable.xcstrings" + "path": "xcode/Localizable.xcstrings" }, { "type": "yaml", - "dictionary": "yaml/[lang].yml" + "path": "yaml/[lang].yml" }, { "type": "yaml-root-key", - "dictionary": "yaml-root-key/[lang].yml" + "path": "yaml-root-key/[lang].yml" }, { "type": "markdown", - "dictionary": "markdown/[lang].md" + "path": "markdown/[lang].md" } ] -} \ No newline at end of file +} diff --git a/packages/cli/src/services/config.ts b/packages/cli/src/services/config.ts index db53e8c74..23b6ec564 100644 --- a/packages/cli/src/services/config.ts +++ b/packages/cli/src/services/config.ts @@ -1,7 +1,7 @@ import Z from 'zod'; import fs from 'fs'; import path from 'path'; -import { projectTypeSchema, sourceLocaleSchema, targetLocaleSchema } from '@replexica/spec'; +import { contentTypeSchema, sourceLocaleSchema, targetLocaleSchema } from '@replexica/spec'; const configFile = "i18n.json"; const configFilePath = path.join(process.cwd(), configFile); @@ -11,16 +11,16 @@ const localeSchema = Z.object({ targets: Z.array(targetLocaleSchema), }); -const projectSchema = Z.object({ - type: projectTypeSchema, - dictionary: Z.string(), +const contentItemSchema = Z.object({ + type: contentTypeSchema, + path: Z.string(), }); const configFileSchema = Z.object({ version: Z.literal(1), debug: Z.boolean().default(false), locale: localeSchema, - projects: Z.array(projectSchema).default([]).optional(), + content: Z.array(contentItemSchema).default([]).optional(), }); export async function loadConfig(): Promise | null> { @@ -42,6 +42,7 @@ export async function createEmptyConfig(): Promise Date: Wed, 1 May 2024 17:58:24 +0200 Subject: [PATCH 08/24] refactor: upd --- packages/cli/src/services/config.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/cli/src/services/config.ts b/packages/cli/src/services/config.ts index 23b6ec564..ea32ef5a7 100644 --- a/packages/cli/src/services/config.ts +++ b/packages/cli/src/services/config.ts @@ -18,7 +18,7 @@ const contentItemSchema = Z.object({ const configFileSchema = Z.object({ version: Z.literal(1), - debug: Z.boolean().default(false), + debug: Z.boolean().default(false).optional(), locale: localeSchema, content: Z.array(contentItemSchema).default([]).optional(), }); @@ -37,7 +37,6 @@ export async function loadConfig(): Promise | n export async function createEmptyConfig(): Promise> { return { version: 1, - debug: false, locale: { source: 'en', targets: ['es'], From c8150dad84bec3740d26ed03573ce9e09fdc0bd9 Mon Sep 17 00:00:00 2001 From: maxprilutskiy Date: Thu, 2 May 2024 12:34:54 +0200 Subject: [PATCH 09/24] chore: upd i18n.json --- demo/classic/i18n.json | 31 ++++++++----------------------- 1 file changed, 8 insertions(+), 23 deletions(-) diff --git a/demo/classic/i18n.json b/demo/classic/i18n.json index 7e4f4a29c..552edb4c8 100644 --- a/demo/classic/i18n.json +++ b/demo/classic/i18n.json @@ -1,32 +1,17 @@ { "version": 1, - "debug": false, "locale": { "source": "en", "targets": [ "es" ] }, - "content": [ - { - "type": "json", - "path": "json/[lang].json" - }, - { - "type": "xcode", - "path": "xcode/Localizable.xcstrings" - }, - { - "type": "yaml", - "path": "yaml/[lang].yml" - }, - { - "type": "yaml-root-key", - "path": "yaml-root-key/[lang].yml" - }, - { - "type": "markdown", - "path": "markdown/[lang].md" - } - ] + "buckets": { + "": "replexica", + "json/[locale].json": "json", + "xcode/Localizable.xcstrings": "xcode", + "yaml/[locale].yml": "yaml", + "yaml-root-key/[locale].yml": "yaml-root-key", + "markdown/[locale].md": "markdown" + } } From f554250beb85fdf67d8b14f8bfc99cc9b08df2b6 Mon Sep 17 00:00:00 2001 From: maxprilutskiy Date: Thu, 2 May 2024 12:35:09 +0200 Subject: [PATCH 10/24] feat: add translator fn implementation --- packages/cli/src/services/translator.ts | 47 +++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 packages/cli/src/services/translator.ts diff --git a/packages/cli/src/services/translator.ts b/packages/cli/src/services/translator.ts new file mode 100644 index 000000000..1b421105a --- /dev/null +++ b/packages/cli/src/services/translator.ts @@ -0,0 +1,47 @@ +import { createId } from "@paralleldrive/cuid2"; + +export type TranslatorFn = { + (sourceLocale: string, targetLocale: string, data: Record, meta: any): Promise>; +}; + +export type CreateTranslatorOptions = { + apiUrl: string; + apiKey: string; + skipCache: boolean; + cacheOnly: boolean; +}; + +export function createTranslator(options: CreateTranslatorOptions): TranslatorFn { + return async (sourceLocale, targetLocale, data, meta) => { + const workflowId = createId(); + const res = await fetch(`${options.apiUrl}/i18n`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${options.apiKey}`, + }, + body: JSON.stringify({ + params: { + workflowId, + cacheOnly: options.cacheOnly, + skipCache: options.skipCache, + }, + locale: { + source: sourceLocale, + target: targetLocale, + }, + meta, + data, + }), + }); + + if (!res.ok) { + const errorText = await res.text(); + throw new Error(errorText); + } + + const payload = await res.json(); + + return payload; + }; +} From 9b220725b45b62a2ccde19eb8d05098af77306d3 Mon Sep 17 00:00:00 2001 From: maxprilutskiy Date: Thu, 2 May 2024 12:35:21 +0200 Subject: [PATCH 11/24] feat: add replexica bucket processor --- packages/cli/src/services/bucket.ts | 127 ++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 packages/cli/src/services/bucket.ts diff --git a/packages/cli/src/services/bucket.ts b/packages/cli/src/services/bucket.ts new file mode 100644 index 000000000..e62af51fd --- /dev/null +++ b/packages/cli/src/services/bucket.ts @@ -0,0 +1,127 @@ +import path from 'path'; +import fs from 'fs'; + +export function createBucketProcessor(bucketType: string, bucketPath: string, translator: BucketTranslatorFn) { + switch (bucketType) { + default: throw new Error(`Unknown bucket type: ${bucketType}`); + case 'replexica': return new ReplexicaBucketProcessor(bucketPath, translator); + } +} + +export type BucketPayload = { + data: Record; + meta: any; +}; + +export type BucketTranslatorFn = { + (sourceLocale: string, targetLocale: string, data: BucketPayload['data'], meta: BucketPayload['meta']): Promise; +} + +export interface IBucketProcessor { + load(locale: string): Promise; + translate(payload: BucketPayload, sourceLocale: string, targetLocale: string): Promise; + save(locale: string, data: BucketPayload['data']): Promise; +} + +export class ReplexicaBucketProcessor implements IBucketProcessor { + constructor( + private bucketPath: string, + private translator: BucketTranslatorFn, + ) { + if (bucketPath !== '') { + throw new Error(`Unknown bucket path: ${bucketPath}. Replexica bucket path must be an empty string: ''.`); + } + } + + async load(locale: string): Promise { + const [ + meta, + data, + ] = await Promise.all([ + this._loadMeta(), + this._loadData(locale), + ]); + + return { data, meta }; + } + + async translate(payload: BucketPayload, sourceLocale: string, targetLocale: string): Promise { + const resultData: any = {}; + for (const [fileId, fileData] of Object.entries(payload.data)) { + const partialLocaleData = { [fileId]: fileData }; + const partialResult = await this.translator( + sourceLocale, + targetLocale, + partialLocaleData, + payload.meta, + ); + resultData[fileId] = partialResult.data[fileId]; + } + + return { + data: resultData, + meta: payload.meta, + }; + } + + async save(locale: string, payload: BucketPayload): Promise { + await Promise.all([ + this._saveFullData(locale, payload), + this._saveClientData(locale, payload), + ]); + } + + private async _loadMeta(): Promise { + const bucketDir = path.resolve(process.cwd(), 'node_modules', '@replexica/translations'); + const metaFilePath = path.resolve(bucketDir, '.replexica.json'); + + const exists = await fs.existsSync(metaFilePath); + if (!exists) { return null; } + + const rawMeta = await fs.readFileSync(metaFilePath, 'utf8'); + const meta = JSON.parse(rawMeta); + + return meta; + } + + private async _loadData(locale: string): Promise { + const bucketDir = path.resolve(process.cwd(), 'node_modules', '@replexica/translations'); + const dataFilePath = path.resolve(bucketDir, `${locale}.json`); + + const exists = await fs.existsSync(dataFilePath); + if (!exists) { return {}; } + + const rawData = await fs.readFileSync(dataFilePath, 'utf8'); + const data = JSON.parse(rawData); + + return data; + } + + private async _saveFullData(locale: string, payload: BucketPayload): Promise { + const bucketDir = path.resolve(process.cwd(), 'node_modules', '@replexica/translations'); + const dataFilePath = path.resolve(bucketDir, `${locale}.json`); + + const content = JSON.stringify(payload.data, null, 2); + await fs.writeFileSync(dataFilePath, content); + } + + private async _saveClientData(locale: string, payload: BucketPayload): Promise { + const bucketDir = path.resolve(process.cwd(), 'node_modules', '@replexica/translations'); + const dataFilePath = path.resolve(bucketDir, `${locale}.client.json`); + + const newData = { + ...payload.data, + }; + + for (const [fileId, fileData] of Object.entries(payload.meta.files || {})) { + const isClient = (fileData as any).isClient; + + if (!isClient) { + delete newData[fileId]; + } + } + + const content = JSON.stringify(newData, null, 2); + await fs.writeFileSync(dataFilePath, content); + } +} From 99da1ae159c77cfca7290b355ac171614b8247a4 Mon Sep 17 00:00:00 2001 From: maxprilutskiy Date: Thu, 2 May 2024 14:40:55 +0200 Subject: [PATCH 12/24] feat: implement replexica bucket processor --- demo/next-app/i18n.json | 12 + packages/cli/src/auth.ts | 12 +- packages/cli/src/i18n.ts | 196 ++++------- packages/cli/src/localize/config/default.ts | 4 +- packages/cli/src/localize/config/schema.ts | 10 +- packages/cli/src/localize/index.ts | 356 ++++++++++---------- packages/cli/src/services/auth.ts | 40 +++ packages/cli/src/services/bucket.ts | 3 +- packages/cli/src/services/check-auth.ts | 44 --- packages/cli/src/services/config.ts | 12 +- packages/cli/src/services/translator.ts | 10 +- 11 files changed, 325 insertions(+), 374 deletions(-) create mode 100644 demo/next-app/i18n.json create mode 100644 packages/cli/src/services/auth.ts delete mode 100644 packages/cli/src/services/check-auth.ts diff --git a/demo/next-app/i18n.json b/demo/next-app/i18n.json new file mode 100644 index 000000000..3bf990eab --- /dev/null +++ b/demo/next-app/i18n.json @@ -0,0 +1,12 @@ +{ + "version": 1, + "locale": { + "source": "en", + "targets": [ + "es" + ] + }, + "buckets": { + "": "replexica" + } +} \ No newline at end of file diff --git a/packages/cli/src/auth.ts b/packages/cli/src/auth.ts index 771463437..2c2563e1d 100644 --- a/packages/cli/src/auth.ts +++ b/packages/cli/src/auth.ts @@ -6,8 +6,8 @@ import open from 'open'; import readline from 'readline/promises'; import { loadSettings } from "./services/settings.js"; import { getEnv } from "./services/env.js"; -import { checkAuth } from "./services/check-auth.js"; import { saveApiKey } from "./services/api-key.js"; +import { loadAuth } from "./services/auth.js"; export default new Command() .command("auth") @@ -28,7 +28,15 @@ export default new Command() config = await loadSettings(); } - await checkAuth(); + const auth = await loadAuth({ + apiUrl: env.REPLEXICA_API_URL, + apiKey: config.auth.apiKey!, + }); + if (!auth) { + Ora().warn('Not authenticated'); + } else { + Ora().succeed(`Authenticated as ${auth.email}`); + } } catch (error: any) { Ora().fail(error.message); process.exit(1); diff --git a/packages/cli/src/i18n.ts b/packages/cli/src/i18n.ts index 67f5b8be9..5d770aa72 100644 --- a/packages/cli/src/i18n.ts +++ b/packages/cli/src/i18n.ts @@ -1,14 +1,12 @@ import { Command } from 'commander'; import Ora from 'ora'; import { getEnv } from './services/env.js'; -import path from 'path'; -import fs from 'fs/promises'; -import { createId } from '@paralleldrive/cuid2'; -import { checkAuth } from './services/check-auth.js'; -import { loadApiKey } from './services/api-key.js'; - -const buildDataDir = path.resolve(process.cwd(), 'node_modules', '@replexica/translations'); -const buildDataFilePath = path.resolve(buildDataDir, '.replexica.json'); +import Z from 'zod'; +import { loadConfig } from './services/config.js'; +import { createBucketProcessor } from './services/bucket.js'; +import { createTranslator } from './services/translator.js'; +import { loadSettings } from './services/settings.js'; +import { loadAuth } from './services/auth.js'; export default new Command() .command('i18n') @@ -17,147 +15,79 @@ export default new Command() .option('--cache-only', 'Only use cached data, and fail if there is new i18n data to process') .option('--skip-cache', 'Skip using cached data and process all i18n data') .action(async (options) => { - const spinner = Ora(); + let spinner = Ora(); try { - if (options.cacheOnly && options.skipCache) { - throw new Error(`Cannot use both --cache-only and --skip-cache options at the same time.`); - } + const flags = await loadFlags(options); + const settings = await loadSettings(); + const env = getEnv(); + const config = await loadConfiguration(); - const authStatus = await checkAuth(); - if (!authStatus) { - throw new Error(`You are not authenticated. Please run 'replexica auth' to authenticate.`); - } - - spinner.start('Loading Replexica build data...'); - const buildData = await loadBuildData(); - if (!buildData) { - throw new Error(`Couldn't load Replexica build data. Did you forget to build your app?`); - } - - const localeSource = buildData.settings?.locale?.source; - if (!localeSource) { - throw new Error(`No source locale found in Replexica build data. Please check your Replexica configuration and try again.`); - } - - const localeTargets = buildData.settings?.locale?.targets || []; - if (!localeTargets.length) { - throw new Error(`No target locales found in Replexica build data. Please check your Replexica configuration and try again.`); - } - - const localeSourceData = await loadLocaleData(localeSource); - if (!localeSourceData) { - throw new Error(`Couldn't load source locale data for source locale ${localeSource}. Did you forget to build your app?`); - } - - spinner.succeed('Replexica data loaded!'); - - const workflowId = createId(); - for (let i = 0; i < localeTargets.length; i++) { - const targetLocale = localeTargets[i]; - const resultData: any = {}; - - const localeEntries = Object.entries(localeSourceData || {}); - for (let j = 0; j < localeEntries.length; j++) { - const [localeFileId, localeFileData] = localeEntries[j]; - spinner.start(`[${targetLocale}] Processing file ${j + 1}/${localeEntries.length}...`); + spinner = Ora().start('Authenticating...'); + const auth = await loadAuth({ + apiUrl: env.REPLEXICA_API_URL, + apiKey: settings.auth.apiKey!, + }); + spinner.succeed(`Authenticated as ${auth.email}.`); + + + const sourceLocale = config.locale.source; + const targetLocales = config.locale.targets; + const bucketEntries = Object.entries(config.buckets!); + + if (!targetLocales.length) { + spinner.warn('No target locales found in configuration. Please add at least one target locale.'); + } else if (!bucketEntries.length) { + spinner.warn('No buckets found in configuration. Please add at least one bucket.'); + } else { + spinner = Ora().start(`Translating ${bucketEntries.length} buckets to ${targetLocales.length} locales...`); + for (const [bucketPath, bucketType] of bucketEntries) { + const bucketSpinner = Ora({ indent: 1 }).start(`Translating ${bucketType} bucket ${bucketPath}...`); + for (const targetLocale of targetLocales) { + const localeSpinner = Ora({ indent: 2 }).start(`Translating from ${sourceLocale} to ${targetLocale}...`); + const translatorFn = createTranslator({ + apiUrl: env.REPLEXICA_API_URL, + apiKey: settings.auth.apiKey!, + skipCache: flags.skipCache, + cacheOnly: flags.cacheOnly, + }); + const processor = createBucketProcessor(bucketType, bucketPath, translatorFn); - const partialLocaleData = { [localeFileId]: localeFileData }; - const result = await processI18n( - { workflowId, cacheOnly: !!options.cacheOnly, skipCache: !!options.skipCache }, - { source: localeSource, target: targetLocale }, - buildData.meta, - partialLocaleData, - ); - resultData[localeFileId] = result.data[localeFileId]; + const translatable = await processor.load(sourceLocale); + const translated = await processor.translate(translatable, sourceLocale, targetLocale); - spinner.succeed(`[${targetLocale}] File ${j + 1}/${localeEntries.length} processed.`); + await processor.save(targetLocale, translated); + localeSpinner.succeed(`Translation from ${sourceLocale} to ${targetLocale} completed.`); + } + bucketSpinner.succeed(`Bucket ${bucketPath} translated.`); } - - await saveFullLocaleData(targetLocale, resultData); - await saveClientLocaleData(targetLocale, resultData, buildData.meta); + spinner.succeed('Translations completed successfully!'); } - - spinner.succeed('Replexica processing complete!'); } catch (error: any) { spinner.fail(error.message); return process.exit(1); } }); -async function processI18n( - params: { workflowId: string, cacheOnly: boolean, skipCache: boolean }, - locale: { source: string, target: string }, - meta: any, - data: any, -) { - const env = getEnv(); - const apiKey = await loadApiKey(); - const res = await fetch(`${env.REPLEXICA_API_URL}/i18n`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${apiKey}`, - }, - body: JSON.stringify({ - params, - locale, - meta, - data, - }), - }); - if (!res.ok) { - const errorText = await res.text(); - throw new Error(errorText); +async function loadFlags(options: any) { + try { + const flags = Z.object({ + cacheOnly: Z.boolean().optional().default(false), + skipCache: Z.boolean().optional().default(false), + }) + .passthrough() + .parse(options); + return flags; + } catch { + throw new Error(`Couldn't parse flags. Please check your input and try again.`) } - const payload = await res.json(); - return payload; -} - -async function loadBuildData() { - const fileExists = await fs.access( - buildDataFilePath, - fs.constants.F_OK, - ).then(() => true).catch(() => false); - if (!fileExists) { return null; } - - const buildDataFile = await fs.readFile(buildDataFilePath, 'utf-8'); - const buildData = JSON.parse(buildDataFile); - return buildData; -} - -async function loadLocaleData(locale: string) { - const localeFilePath = path.resolve(buildDataDir, `${locale}.json`); - const fileExists = await fs.access( - localeFilePath, - fs.constants.F_OK, - ).then(() => true).catch(() => false); - if (!fileExists) { return null; } - - const localeFile = await fs.readFile(localeFilePath, 'utf-8'); - const localeData = JSON.parse(localeFile); - return localeData; -} - -async function saveFullLocaleData(locale: string, data: any) { - const localeFilePath = path.resolve(buildDataDir, `${locale}.json`); - await fs.writeFile(localeFilePath, JSON.stringify(data, null, 2)); } -async function saveClientLocaleData(locale: string, data: any, meta: any) { - const newData = { - ...data, - }; - - for (const [fileId, fileData] of Object.entries(meta.files || {})) { - const isClient = (fileData as any).isClient; - - if (!isClient) { - delete newData[fileId]; - } +async function loadConfiguration() { + const config = await loadConfig(); + if (!config) { + throw new Error(`Couldn't load i18n configuration. Please run 'replexica init' to initialize your Replexica project.`); } - - const localeFilePath = path.resolve(buildDataDir, `${locale}.client.json`); - await fs.writeFile(localeFilePath, JSON.stringify(newData, null, 2)); + return config; } diff --git a/packages/cli/src/localize/config/default.ts b/packages/cli/src/localize/config/default.ts index 239d49fad..80b97a03c 100644 --- a/packages/cli/src/localize/config/default.ts +++ b/packages/cli/src/localize/config/default.ts @@ -10,9 +10,9 @@ export const defaultPaths = { export const defaultConfig: ConfigSchema = { version: 1, - languages: { source: 'en', target: ['es', 'fr'] }, + languages: { source: 'en', target: ['es'] }, projects: [ - { name: 'demo', type: 'json', dictionary: defaultPaths.dictionaryPattern }, + { name: 'demo', type: 'json', path: defaultPaths.dictionaryPattern }, ], }; diff --git a/packages/cli/src/localize/config/schema.ts b/packages/cli/src/localize/config/schema.ts index 760fa987e..f32257c18 100644 --- a/packages/cli/src/localize/config/schema.ts +++ b/packages/cli/src/localize/config/schema.ts @@ -1,21 +1,21 @@ import Z from 'zod'; -import { sourceLocaleSchema, targetLocaleSchema, projectTypeSchema } from '@replexica/spec'; +import { sourceLocaleSchema, targetLocaleSchema, contentTypeSchema } from '@replexica/spec'; const languageSchema = Z.object({ source: sourceLocaleSchema, target: Z.array(targetLocaleSchema), }); -const projectSchema = Z.object({ +const contentItemSchema = Z.object({ name: Z.string(), - type: projectTypeSchema.optional().default('json'), - dictionary: Z.string(), + type: contentTypeSchema.optional().default('json'), + path: Z.string(), }); export const configSchema = Z.object({ version: Z.literal(1), languages: languageSchema, - projects: Z.array(projectSchema), + projects: Z.array(contentItemSchema).default([]).optional(), }); export type ConfigSchema = Z.infer; diff --git a/packages/cli/src/localize/index.ts b/packages/cli/src/localize/index.ts index 4d8576848..c7c116cc4 100644 --- a/packages/cli/src/localize/index.ts +++ b/packages/cli/src/localize/index.ts @@ -31,181 +31,181 @@ export default new Command() .argument('[root]', 'Root directory of the repository, containing the .replexica/config.yml config file.', '.') .option('--trigger-type ', 'Environment from which the localization is triggered', 'cli') .option('--trigger-name ', 'Name of the trigger', '') - .action(async (root, options) => { - const config = await extractConfig(root, options.triggerType, options.triggerName); - for (const project of config.projects) { - const sourceLangData = await loadProjectLangData(project, config.sourceLang); - const changedKeys = await calculateChangedKeys(project.name, sourceLangData); - - // Write hash file at the beginning of the process - // So that if it fails in the middle, we won't have - // to re-translate everything from scratch again - await writeHashFile(project.name, sourceLangData); - - for (const targetLang of config.targetLangs) { - const targetLangData = await loadProjectLangData(project, targetLang); - - const removedKeys = _.difference(Object.keys(targetLangData), Object.keys(sourceLangData)); - const missingKeys = _.difference(Object.keys(sourceLangData), Object.keys(targetLangData)); - - const projectLogPrefix = `[${project.name}]`; - console.info(`${projectLogPrefix} Removed: ${removedKeys.length}. Changed: ${changedKeys.length}. Missing: ${missingKeys.length}.`); - - const keysToTranslate = _.uniq([...changedKeys, ...missingKeys]); - - const translationLogPrefix = `${projectLogPrefix} (${config.sourceLang} -> ${targetLang})`; - console.log(`${translationLogPrefix} Translating ${keysToTranslate.length} keys`); - - let langDataUpdate: Record = {}; - if (keysToTranslate.length) { - const keysToTranslateChunks = _.chunk(keysToTranslate, TRANSLATIONS_PER_BATCH); - - let translatedKeysCount = 0; - const groupId = `leg_${createId()}`; - for (const keysToTranslateChunk of keysToTranslateChunks) { - console.log(`${translationLogPrefix} Translating keys, ${translatedKeysCount}/${keysToTranslate.length}`); - const partialDiffRecord = _.pick(sourceLangData, keysToTranslateChunk); - const partialLangDataUpdate = await translateRecord( - config.sourceLang, - targetLang, - partialDiffRecord, - groupId, - ); - langDataUpdate = _.merge(langDataUpdate, partialLangDataUpdate); - - translatedKeysCount += keysToTranslateChunk.length; - } - - console.log(`Done`); - } else { - console.log(`Skipped`); - } - - const newTargetLangData = _.chain(targetLangData) - .merge(langDataUpdate) - .omit(removedKeys) - .value(); - - await saveProjectLangData(project, targetLang, newTargetLangData); - } - } - }); -async function writeHashFile(projectName: string, sourceLangData: Record) { - const projectHashfileNode: Record = {}; - for (const [key, value] of Object.entries(sourceLangData)) { - const valueHash = Crypto - .createHash('sha256') - .update(value) - .digest('hex'); - - projectHashfileNode[key] = valueHash; - } - const replexicaHashfileContent = await fs.readFile('.replexica/hash.yaml', 'utf-8') - .catch(() => '') - .then((content) => content.trim() || ''); - const replexicaHashfile = YAML.parse(replexicaHashfileContent) || {} as Record; - replexicaHashfile.version = replexicaHashfile.version || 1; - replexicaHashfile[projectName] = projectHashfileNode; - - const newReplexicaHashfileContent = [ - '# DO NOT MODIFY THIS FILE MANUALLY', - '# This file is auto-generated by Replexica. Please keep it in your version control system.', - YAML.stringify(replexicaHashfile), - ].join('\n'); - await fs.writeFile('.replexica/hash.yaml', newReplexicaHashfileContent); -} - -async function calculateChangedKeys(projectName: string, sourceLangData: Record): Promise { - const replexicaHashfileContent = await fs.readFile('.replexica/hash.yaml', 'utf-8') - .catch(() => '') - .then((content) => content.trim() || ''); - const replexicaHashfile = YAML.parse(replexicaHashfileContent) || {} as Record; - const projectHashfileNode = replexicaHashfile[projectName] || {}; - - const result: string[] = []; - for (const [key, value] of Object.entries(sourceLangData)) { - const valueHash = Crypto - .createHash('sha256') - .update(value) - .digest('hex'); - - if (projectHashfileNode[key] !== valueHash) { - result.push(key); - } - } - - return result; -} - -async function loadProjectLangData(project: ConfigSchema['projects'][0], lang: string): Promise> { - const processor = langDataProcessorsMap.get(project.type); - if (!processor) { throw new Error('Unsupported project type ' + project.type); } - - const result = await processor.loadLangJson(project.dictionary, lang); - - return result; -} - -async function saveProjectLangData(project: ConfigSchema['projects'][0], lang: string, data: Record) { - const processor = langDataProcessorsMap.get(project.type); - if (!processor) { throw new Error('Unsupported project type ' + project.type); } - - await processor.saveLangJson(project.dictionary, lang, data); -} - -async function extractConfig(root: string, triggerType: string, triggerName: string) { - const configRoot = path.resolve(process.cwd(), root); - const configFilePath = path.join(configRoot, '.replexica/config.yml'); - - const configFileExists = await fs.stat(configFilePath).then(() => true).catch(() => false); - if (!configFileExists) { - throw new Error(`Config file not found at ${configFilePath}.`); - } - - const config: ConfigSchema | null = configFileExists ? loadConfig(configFilePath) : null; - - const sourceLang = config?.languages.source; - if (!sourceLang) { - throw new Error('Source language must be specified.'); - } - - const targetLangs = (config?.languages.target || []).filter(Boolean); - if (targetLangs.length === 0) { - throw new Error('At least one target language must be specified.'); - } - - const projects = config?.projects || []; - if (projects.length === 0) { - throw new Error('At least one project must be specified.'); - } - - return { - sourceLang, - targetLangs, - projects, - triggerType, - triggerName, - }; -} - -async function translateRecord( - sourceLang: string, - targetLang: string, - data: Record, - groupId: string, -): Promise> { - if (Object.keys(data).length === 0) { return {}; } - - const replexica = getReplexicaClient(); - const translateRecordResponse = await replexica.localizeJson({ - groupId, - triggerType: 'cli', - triggerName: 'cli', - - sourceLocale: sourceLang, - targetLocale: targetLang, - data, - }); - - return translateRecordResponse.data; -} +// .action(async (root, options) => { +// const config = await extractConfig(root, options.triggerType, options.triggerName); +// for (const project of config.projects) { +// const sourceLangData = await loadProjectLangData(project, config.sourceLang); +// const changedKeys = await calculateChangedKeys(project.name, sourceLangData); + +// // Write hash file at the beginning of the process +// // So that if it fails in the middle, we won't have +// // to re-translate everything from scratch again +// await writeHashFile(project.name, sourceLangData); + +// for (const targetLang of config.targetLangs) { +// const targetLangData = await loadProjectLangData(project, targetLang); + +// const removedKeys = _.difference(Object.keys(targetLangData), Object.keys(sourceLangData)); +// const missingKeys = _.difference(Object.keys(sourceLangData), Object.keys(targetLangData)); + +// const projectLogPrefix = `[${project.name}]`; +// console.info(`${projectLogPrefix} Removed: ${removedKeys.length}. Changed: ${changedKeys.length}. Missing: ${missingKeys.length}.`); + +// const keysToTranslate = _.uniq([...changedKeys, ...missingKeys]); + +// const translationLogPrefix = `${projectLogPrefix} (${config.sourceLang} -> ${targetLang})`; +// console.log(`${translationLogPrefix} Translating ${keysToTranslate.length} keys`); + +// let langDataUpdate: Record = {}; +// if (keysToTranslate.length) { +// const keysToTranslateChunks = _.chunk(keysToTranslate, TRANSLATIONS_PER_BATCH); + +// let translatedKeysCount = 0; +// const groupId = `leg_${createId()}`; +// for (const keysToTranslateChunk of keysToTranslateChunks) { +// console.log(`${translationLogPrefix} Translating keys, ${translatedKeysCount}/${keysToTranslate.length}`); +// const partialDiffRecord = _.pick(sourceLangData, keysToTranslateChunk); +// const partialLangDataUpdate = await translateRecord( +// config.sourceLang, +// targetLang, +// partialDiffRecord, +// groupId, +// ); +// langDataUpdate = _.merge(langDataUpdate, partialLangDataUpdate); + +// translatedKeysCount += keysToTranslateChunk.length; +// } + +// console.log(`Done`); +// } else { +// console.log(`Skipped`); +// } + +// const newTargetLangData = _.chain(targetLangData) +// .merge(langDataUpdate) +// .omit(removedKeys) +// .value(); + +// await saveProjectLangData(project, targetLang, newTargetLangData); +// } +// } +// }); +// async function writeHashFile(projectName: string, sourceLangData: Record) { +// const projectHashfileNode: Record = {}; +// for (const [key, value] of Object.entries(sourceLangData)) { +// const valueHash = Crypto +// .createHash('sha256') +// .update(value) +// .digest('hex'); + +// projectHashfileNode[key] = valueHash; +// } +// const replexicaHashfileContent = await fs.readFile('.replexica/hash.yaml', 'utf-8') +// .catch(() => '') +// .then((content) => content.trim() || ''); +// const replexicaHashfile = YAML.parse(replexicaHashfileContent) || {} as Record; +// replexicaHashfile.version = replexicaHashfile.version || 1; +// replexicaHashfile[projectName] = projectHashfileNode; + +// const newReplexicaHashfileContent = [ +// '# DO NOT MODIFY THIS FILE MANUALLY', +// '# This file is auto-generated by Replexica. Please keep it in your version control system.', +// YAML.stringify(replexicaHashfile), +// ].join('\n'); +// await fs.writeFile('.replexica/hash.yaml', newReplexicaHashfileContent); +// } + +// async function calculateChangedKeys(projectName: string, sourceLangData: Record): Promise { +// const replexicaHashfileContent = await fs.readFile('.replexica/hash.yaml', 'utf-8') +// .catch(() => '') +// .then((content) => content.trim() || ''); +// const replexicaHashfile = YAML.parse(replexicaHashfileContent) || {} as Record; +// const projectHashfileNode = replexicaHashfile[projectName] || {}; + +// const result: string[] = []; +// for (const [key, value] of Object.entries(sourceLangData)) { +// const valueHash = Crypto +// .createHash('sha256') +// .update(value) +// .digest('hex'); + +// if (projectHashfileNode[key] !== valueHash) { +// result.push(key); +// } +// } + +// return result; +// } + +// async function loadProjectLangData(project: ConfigSchema['projects'][0], lang: string): Promise> { +// const processor = langDataProcessorsMap.get(project.type); +// if (!processor) { throw new Error('Unsupported project type ' + project.type); } + +// const result = await processor.loadLangJson(project.dictionary, lang); + +// return result; +// } + +// async function saveProjectLangData(project: ConfigSchema['projects'][0], lang: string, data: Record) { +// const processor = langDataProcessorsMap.get(project.type); +// if (!processor) { throw new Error('Unsupported project type ' + project.type); } + +// await processor.saveLangJson(project.dictionary, lang, data); +// } + +// async function extractConfig(root: string, triggerType: string, triggerName: string) { +// const configRoot = path.resolve(process.cwd(), root); +// const configFilePath = path.join(configRoot, '.replexica/config.yml'); + +// const configFileExists = await fs.stat(configFilePath).then(() => true).catch(() => false); +// if (!configFileExists) { +// throw new Error(`Config file not found at ${configFilePath}.`); +// } + +// const config: ConfigSchema | null = configFileExists ? loadConfig(configFilePath) : null; + +// const sourceLang = config?.languages.source; +// if (!sourceLang) { +// throw new Error('Source language must be specified.'); +// } + +// const targetLangs = (config?.languages.target || []).filter(Boolean); +// if (targetLangs.length === 0) { +// throw new Error('At least one target language must be specified.'); +// } + +// const projects = config?.projects || []; +// if (projects.length === 0) { +// throw new Error('At least one project must be specified.'); +// } + +// return { +// sourceLang, +// targetLangs, +// projects, +// triggerType, +// triggerName, +// }; +// } + +// async function translateRecord( +// sourceLang: string, +// targetLang: string, +// data: Record, +// groupId: string, +// ): Promise> { +// if (Object.keys(data).length === 0) { return {}; } + +// const replexica = getReplexicaClient(); +// const translateRecordResponse = await replexica.localizeJson({ +// groupId, +// triggerType: 'cli', +// triggerName: 'cli', + +// sourceLocale: sourceLang, +// targetLocale: targetLang, +// data, +// }); + +// return translateRecordResponse.data; +// } diff --git a/packages/cli/src/services/auth.ts b/packages/cli/src/services/auth.ts new file mode 100644 index 000000000..57209064b --- /dev/null +++ b/packages/cli/src/services/auth.ts @@ -0,0 +1,40 @@ +export type LoadAuthParams = { + apiUrl: string; + apiKey: string; +}; + +export async function loadAuth(params: LoadAuthParams) { + const whoami = await fetchWhoami(params.apiUrl, params.apiKey); + if (!whoami) { + throw new Error("Failed to authenticate"); + } + + return { + email: whoami.email, + }; +} + +async function fetchWhoami(apiUrl: string, apiKey: string | null) { + try { + const res = await fetch(`${apiUrl}/whoami`, { + method: "POST", + headers: { + Authorization: `Bearer ${apiKey}`, + ContentType: "application/json", + }, + }); + + if (res.ok) { + return res.json(); + } + + return null; + } catch (error) { + const isNetworkError = error instanceof TypeError && error.message === "fetch failed"; + if (isNetworkError) { + throw new Error(`Failed to connect to the API at ${apiUrl}. Please check your connection and try again.`); + } else { + throw error; + } + } +} diff --git a/packages/cli/src/services/bucket.ts b/packages/cli/src/services/bucket.ts index e62af51fd..8886740b3 100644 --- a/packages/cli/src/services/bucket.ts +++ b/packages/cli/src/services/bucket.ts @@ -79,7 +79,8 @@ export class ReplexicaBucketProcessor implements IBucketProcessor { if (!exists) { return null; } const rawMeta = await fs.readFileSync(metaFilePath, 'utf8'); - const meta = JSON.parse(rawMeta); + const metaObj = JSON.parse(rawMeta); + const meta = metaObj?.meta || {}; return meta; } diff --git a/packages/cli/src/services/check-auth.ts b/packages/cli/src/services/check-auth.ts deleted file mode 100644 index d800e8e16..000000000 --- a/packages/cli/src/services/check-auth.ts +++ /dev/null @@ -1,44 +0,0 @@ -import Ora from "ora"; - -import { getEnv } from "./env.js"; -import { loadSettings } from "./settings.js"; - -export async function checkAuth() { - const env = getEnv(); - const settings = await loadSettings(); - - const finalApiKey = env.REPLEXICA_API_KEY || settings.auth.apiKey; - const isApiKeyFromEnv = !!env.REPLEXICA_API_KEY; - - const spinner = Ora().start('Checking login status'); - - const whoami = await fetchWhoami(env.REPLEXICA_API_URL, finalApiKey); - if (!whoami) { - spinner.warn('Not logged in. Please run `replexica auth --login` to authenticate.'); - return false; - } - - let msg = `Logged in as ${whoami.email}`; - if (isApiKeyFromEnv) { - msg += ' (via REPLEXICA_API_KEY from environment)'; - } - spinner.succeed(msg); - - return true; -} - -async function fetchWhoami(apiUrl: string, apiKey: string | null) { - const res = await fetch(`${apiUrl}/whoami`, { - method: "POST", - headers: { - Authorization: `Bearer ${apiKey}`, - ContentType: "application/json", - }, - }); - - if (res.ok) { - return res.json(); - } - - return null; -} \ No newline at end of file diff --git a/packages/cli/src/services/config.ts b/packages/cli/src/services/config.ts index ea32ef5a7..4735aebe9 100644 --- a/packages/cli/src/services/config.ts +++ b/packages/cli/src/services/config.ts @@ -11,16 +11,16 @@ const localeSchema = Z.object({ targets: Z.array(targetLocaleSchema), }); -const contentItemSchema = Z.object({ - type: contentTypeSchema, - path: Z.string(), -}); +const bucketsSchema = Z.record( + Z.string(), + Z.union([Z.literal('replexica'), contentTypeSchema]), +); const configFileSchema = Z.object({ version: Z.literal(1), debug: Z.boolean().default(false).optional(), locale: localeSchema, - content: Z.array(contentItemSchema).default([]).optional(), + buckets: bucketsSchema.default({}).optional(), }); export async function loadConfig(): Promise | null> { @@ -41,7 +41,7 @@ export async function createEmptyConfig(): Promise Date: Thu, 2 May 2024 15:56:11 +0200 Subject: [PATCH 13/24] refactor: refactoring --- demo/next-app/i18n.json | 2 +- packages/cli/src/i18n.ts | 13 +++-- packages/cli/src/services/bucket/core.ts | 36 +++++++++++++ packages/cli/src/services/bucket/json.ts | 6 +++ packages/cli/src/services/bucket/jsonlike.ts | 53 +++++++++++++++++++ .../{bucket.ts => bucket/replexica.ts} | 32 +++-------- packages/cli/src/services/config.ts | 10 ++-- 7 files changed, 114 insertions(+), 38 deletions(-) create mode 100644 packages/cli/src/services/bucket/core.ts create mode 100644 packages/cli/src/services/bucket/json.ts create mode 100644 packages/cli/src/services/bucket/jsonlike.ts rename packages/cli/src/services/{bucket.ts => bucket/replexica.ts} (78%) diff --git a/demo/next-app/i18n.json b/demo/next-app/i18n.json index 3bf990eab..dc53323fa 100644 --- a/demo/next-app/i18n.json +++ b/demo/next-app/i18n.json @@ -9,4 +9,4 @@ "buckets": { "": "replexica" } -} \ No newline at end of file +} diff --git a/packages/cli/src/i18n.ts b/packages/cli/src/i18n.ts index 5d770aa72..455521c80 100644 --- a/packages/cli/src/i18n.ts +++ b/packages/cli/src/i18n.ts @@ -3,7 +3,7 @@ import Ora from 'ora'; import { getEnv } from './services/env.js'; import Z from 'zod'; import { loadConfig } from './services/config.js'; -import { createBucketProcessor } from './services/bucket.js'; +import { createBucketProcessor } from './services/bucket/core.js'; import { createTranslator } from './services/translator.js'; import { loadSettings } from './services/settings.js'; import { loadAuth } from './services/auth.js'; @@ -42,9 +42,12 @@ export default new Command() } else { spinner = Ora().start(`Translating ${bucketEntries.length} buckets to ${targetLocales.length} locales...`); for (const [bucketPath, bucketType] of bucketEntries) { - const bucketSpinner = Ora({ indent: 1 }).start(`Translating ${bucketType} bucket ${bucketPath}...`); + let spinnerPrefix = `[${bucketType}]`; + if (bucketPath) { spinnerPrefix += `(${bucketPath})`; } + const bucketSpinner = Ora({ prefixText: spinnerPrefix }); + for (const targetLocale of targetLocales) { - const localeSpinner = Ora({ indent: 2 }).start(`Translating from ${sourceLocale} to ${targetLocale}...`); + bucketSpinner.start(`Translating from ${sourceLocale} to ${targetLocale}...`); const translatorFn = createTranslator({ apiUrl: env.REPLEXICA_API_URL, apiKey: settings.auth.apiKey!, @@ -57,9 +60,9 @@ export default new Command() const translated = await processor.translate(translatable, sourceLocale, targetLocale); await processor.save(targetLocale, translated); - localeSpinner.succeed(`Translation from ${sourceLocale} to ${targetLocale} completed.`); + bucketSpinner.succeed(`Translation from ${sourceLocale} to ${targetLocale} completed.`); } - bucketSpinner.succeed(`Bucket ${bucketPath} translated.`); + bucketSpinner.succeed(`Bucket translated.`); } spinner.succeed('Translations completed successfully!'); } diff --git a/packages/cli/src/services/bucket/core.ts b/packages/cli/src/services/bucket/core.ts new file mode 100644 index 000000000..66b72ac07 --- /dev/null +++ b/packages/cli/src/services/bucket/core.ts @@ -0,0 +1,36 @@ +import _ from 'lodash'; +import Z from 'zod'; +import { ReplexicaBucketProcessor } from './replexica.js'; +import { contentTypes, contentTypeSchema } from '@replexica/spec'; +import { JsonBucketProcessor } from './json.js'; + +export const bucketTypes = [...contentTypes, 'replexica'] as const; + +export const bucketTypeSchema = Z.union([Z.literal('replexica'), contentTypeSchema]); + +export function createBucketProcessor( + bucketType: typeof bucketTypeSchema._type, + bucketPath: string, + translator: BucketTranslatorFn, +): IBucketProcessor { + switch (bucketType) { + default: throw new Error(`Unknown bucket type: ${bucketType}`); + case 'replexica': return new ReplexicaBucketProcessor(bucketPath, translator); + case 'json': return new JsonBucketProcessor(bucketPath, translator); + } +} + +export type BucketPayload = { + data: Record; + meta: any; +}; + +export type BucketTranslatorFn = { + (sourceLocale: string, targetLocale: string, data: BucketPayload['data'], meta: BucketPayload['meta']): Promise; +} + +export interface IBucketProcessor { + load(locale: string): Promise; + translate(payload: BucketPayload, sourceLocale: string, targetLocale: string): Promise; + save(locale: string, data: BucketPayload['data']): Promise; +} diff --git a/packages/cli/src/services/bucket/json.ts b/packages/cli/src/services/bucket/json.ts new file mode 100644 index 000000000..9e505e50b --- /dev/null +++ b/packages/cli/src/services/bucket/json.ts @@ -0,0 +1,6 @@ +import { IBucketProcessor } from "./core.js"; +import { JsonLikeBucketProcessor } from "./jsonlike.js"; + +export class JsonBucketProcessor extends JsonLikeBucketProcessor implements IBucketProcessor { + +} diff --git a/packages/cli/src/services/bucket/jsonlike.ts b/packages/cli/src/services/bucket/jsonlike.ts new file mode 100644 index 000000000..3581835e2 --- /dev/null +++ b/packages/cli/src/services/bucket/jsonlike.ts @@ -0,0 +1,53 @@ +import _ from "lodash"; +import fs from 'fs'; +import { BucketPayload, BucketTranslatorFn, IBucketProcessor } from "./core.js"; + +export abstract class JsonLikeBucketProcessor implements IBucketProcessor { + constructor( + private bucketPath: string, + private translator: BucketTranslatorFn, + ) { + if (!bucketPath.includes('[lang]')) { + throw new Error(`Invalid bucket path: ${bucketPath}. The path must include the [lang] placeholder.`); + } + if (!bucketPath.endsWith('.json')) { + throw new Error(`Invalid bucket path: ${bucketPath}. The path must have a .json file extension.`); + } + } + + async load(locale: string): Promise { + const filePath = this.bucketPath.replace('[lang]', locale); + const exists = await fs.existsSync(filePath); + if (!exists) { return { data: {}, meta: null }; } + + const rawContent = await fs.readFileSync(filePath, 'utf8'); + const data = JSON.parse(rawContent); + + return { + data, + meta: null, + }; + } + + async translate(payload: BucketPayload, sourceLocale: string, targetLocale: string): Promise> { + // The data contains key-value pairs, so let's translate + // the values in batches of 20 keys max. + const resultData: Record = {}; + + const keys = Object.keys(payload.data); + const batches = _.chunk(keys, 20); + for (const batch of batches) { + const partialData = _.pick(payload.data, batch); + const partialResult = await this.translator(sourceLocale, targetLocale, partialData, payload.meta); + _.merge(resultData, partialResult); + } + + return resultData; + } + + async save(locale: string, data: Record): Promise { + const filePath = this.bucketPath.replace('[lang]', locale); + const content = JSON.stringify(data, null, 2); + await fs.writeFileSync(filePath, content); + } +} diff --git a/packages/cli/src/services/bucket.ts b/packages/cli/src/services/bucket/replexica.ts similarity index 78% rename from packages/cli/src/services/bucket.ts rename to packages/cli/src/services/bucket/replexica.ts index 8886740b3..139aa6bce 100644 --- a/packages/cli/src/services/bucket.ts +++ b/packages/cli/src/services/bucket/replexica.ts @@ -1,27 +1,6 @@ -import path from 'path'; -import fs from 'fs'; - -export function createBucketProcessor(bucketType: string, bucketPath: string, translator: BucketTranslatorFn) { - switch (bucketType) { - default: throw new Error(`Unknown bucket type: ${bucketType}`); - case 'replexica': return new ReplexicaBucketProcessor(bucketPath, translator); - } -} - -export type BucketPayload = { - data: Record; - meta: any; -}; - -export type BucketTranslatorFn = { - (sourceLocale: string, targetLocale: string, data: BucketPayload['data'], meta: BucketPayload['meta']): Promise; -} - -export interface IBucketProcessor { - load(locale: string): Promise; - translate(payload: BucketPayload, sourceLocale: string, targetLocale: string): Promise; - save(locale: string, data: BucketPayload['data']): Promise; -} +import path from "path"; +import fs from "fs"; +import { IBucketProcessor, BucketTranslatorFn, BucketPayload } from "./core.js"; export class ReplexicaBucketProcessor implements IBucketProcessor { constructor( @@ -47,6 +26,9 @@ export class ReplexicaBucketProcessor implements IBucketProcessor { async translate(payload: BucketPayload, sourceLocale: string, targetLocale: string): Promise { const resultData: any = {}; + // Currently the split is done by fileId, but as files can + // get quite large, we might want to split by a certain number + // of files' scopes instead. for (const [fileId, fileData] of Object.entries(payload.data)) { const partialLocaleData = { [fileId]: fileData }; const partialResult = await this.translator( @@ -125,4 +107,4 @@ export class ReplexicaBucketProcessor implements IBucketProcessor { const content = JSON.stringify(newData, null, 2); await fs.writeFileSync(dataFilePath, content); } -} +} \ No newline at end of file diff --git a/packages/cli/src/services/config.ts b/packages/cli/src/services/config.ts index 4735aebe9..998cf562b 100644 --- a/packages/cli/src/services/config.ts +++ b/packages/cli/src/services/config.ts @@ -1,7 +1,8 @@ import Z from 'zod'; import fs from 'fs'; import path from 'path'; -import { contentTypeSchema, sourceLocaleSchema, targetLocaleSchema } from '@replexica/spec'; +import { sourceLocaleSchema, targetLocaleSchema } from '@replexica/spec'; +import { bucketTypeSchema } from './bucket/core.js'; const configFile = "i18n.json"; const configFilePath = path.join(process.cwd(), configFile); @@ -11,16 +12,11 @@ const localeSchema = Z.object({ targets: Z.array(targetLocaleSchema), }); -const bucketsSchema = Z.record( - Z.string(), - Z.union([Z.literal('replexica'), contentTypeSchema]), -); - const configFileSchema = Z.object({ version: Z.literal(1), debug: Z.boolean().default(false).optional(), locale: localeSchema, - buckets: bucketsSchema.default({}).optional(), + buckets: Z.record(Z.string(), bucketTypeSchema).default({}).optional(), }); export async function loadConfig(): Promise | null> { From de8693a0e316de6f55af4289bf75790f00a65424 Mon Sep 17 00:00:00 2001 From: maxprilutskiy Date: Thu, 2 May 2024 16:11:50 +0200 Subject: [PATCH 14/24] refactor: refactoring --- packages/cli/src/i18n.ts | 3 +- packages/cli/src/services/bucket/core.ts | 73 ++++++++++++++++--- packages/cli/src/services/bucket/jsonlike.ts | 29 +++++--- packages/cli/src/services/bucket/replexica.ts | 5 +- packages/cli/src/services/translator.ts | 51 ------------- 5 files changed, 87 insertions(+), 74 deletions(-) delete mode 100644 packages/cli/src/services/translator.ts diff --git a/packages/cli/src/i18n.ts b/packages/cli/src/i18n.ts index 455521c80..e5df1f8ab 100644 --- a/packages/cli/src/i18n.ts +++ b/packages/cli/src/i18n.ts @@ -3,8 +3,7 @@ import Ora from 'ora'; import { getEnv } from './services/env.js'; import Z from 'zod'; import { loadConfig } from './services/config.js'; -import { createBucketProcessor } from './services/bucket/core.js'; -import { createTranslator } from './services/translator.js'; +import { createBucketProcessor, createTranslator } from './services/bucket/core.js'; import { loadSettings } from './services/settings.js'; import { loadAuth } from './services/auth.js'; diff --git a/packages/cli/src/services/bucket/core.ts b/packages/cli/src/services/bucket/core.ts index 66b72ac07..0183bf938 100644 --- a/packages/cli/src/services/bucket/core.ts +++ b/packages/cli/src/services/bucket/core.ts @@ -3,6 +3,25 @@ import Z from 'zod'; import { ReplexicaBucketProcessor } from './replexica.js'; import { contentTypes, contentTypeSchema } from '@replexica/spec'; import { JsonBucketProcessor } from './json.js'; +import { createId } from '@paralleldrive/cuid2'; + +// Bucket processor + +export type BucketPayload = { + data: Record; + meta: any; +}; + +export type BucketTranslatorFn = { + (sourceLocale: string, targetLocale: string, payload: BucketPayload): Promise; +} + +export interface IBucketProcessor { + load(locale: string): Promise; + translate(payload: BucketPayload, sourceLocale: string, targetLocale: string): Promise; + save(locale: string, payload: BucketPayload): Promise; +} + export const bucketTypes = [...contentTypes, 'replexica'] as const; @@ -20,17 +39,51 @@ export function createBucketProcessor( } } -export type BucketPayload = { - data: Record; - meta: any; + +// Translator + + +export type CreateTranslatorOptions = { + apiUrl: string; + apiKey: string; + skipCache: boolean; + cacheOnly: boolean; }; -export type BucketTranslatorFn = { - (sourceLocale: string, targetLocale: string, data: BucketPayload['data'], meta: BucketPayload['meta']): Promise; -} +export function createTranslator(options: CreateTranslatorOptions): BucketTranslatorFn { + return async (sourceLocale, targetLocale, payload) => { + const workflowId = createId(); + const res = await fetch(`${options.apiUrl}/i18n`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${options.apiKey}`, + }, + body: JSON.stringify({ + params: { + workflowId, + cacheOnly: options.cacheOnly, + skipCache: options.skipCache, + }, + locale: { + source: sourceLocale, + target: targetLocale, + }, + meta: payload.meta, + data: payload.data, + }, null, 2), + }); -export interface IBucketProcessor { - load(locale: string): Promise; - translate(payload: BucketPayload, sourceLocale: string, targetLocale: string): Promise; - save(locale: string, data: BucketPayload['data']): Promise; + if (!res.ok) { + if (res.status === 400) { + throw new Error(`Invalid request: ${res.statusText}`); + } else { + const errorText = await res.text(); + throw new Error(errorText); + } + } + + const result = await res.json(); + return result; + }; } diff --git a/packages/cli/src/services/bucket/jsonlike.ts b/packages/cli/src/services/bucket/jsonlike.ts index 3581835e2..2680f2df7 100644 --- a/packages/cli/src/services/bucket/jsonlike.ts +++ b/packages/cli/src/services/bucket/jsonlike.ts @@ -23,13 +23,16 @@ export abstract class JsonLikeBucketProcessor implements IBucketProcessor { const rawContent = await fs.readFileSync(filePath, 'utf8'); const data = JSON.parse(rawContent); - return { - data, - meta: null, - }; + const rawResult = { data, meta: null }; + const result = await this.postLoad(rawResult, locale); + return result; } - async translate(payload: BucketPayload, sourceLocale: string, targetLocale: string): Promise> { + async postLoad(payload: BucketPayload, locale: string): Promise { + return payload; + } + + async translate(payload: BucketPayload, sourceLocale: string, targetLocale: string): Promise { // The data contains key-value pairs, so let's translate // the values in batches of 20 keys max. const resultData: Record = {}; @@ -38,16 +41,24 @@ export abstract class JsonLikeBucketProcessor implements IBucketProcessor { const batches = _.chunk(keys, 20); for (const batch of batches) { const partialData = _.pick(payload.data, batch); - const partialResult = await this.translator(sourceLocale, targetLocale, partialData, payload.meta); + const partialPayload = { data: partialData, meta: payload.meta }; + const partialResult = await this.translator(sourceLocale, targetLocale, partialPayload); _.merge(resultData, partialResult); } - return resultData; + const result = { data: resultData, meta: payload.meta }; + return result; } - async save(locale: string, data: Record): Promise { + async preSave(payload: BucketPayload, locale: string): Promise { + return payload; + } + + async save(locale: string, rawPayload: BucketPayload): Promise { + const payload = await this.preSave(rawPayload, locale); + const filePath = this.bucketPath.replace('[lang]', locale); - const content = JSON.stringify(data, null, 2); + const content = JSON.stringify(payload.data, null, 2); await fs.writeFileSync(filePath, content); } } diff --git a/packages/cli/src/services/bucket/replexica.ts b/packages/cli/src/services/bucket/replexica.ts index 139aa6bce..4d45e960a 100644 --- a/packages/cli/src/services/bucket/replexica.ts +++ b/packages/cli/src/services/bucket/replexica.ts @@ -31,11 +31,12 @@ export class ReplexicaBucketProcessor implements IBucketProcessor { // of files' scopes instead. for (const [fileId, fileData] of Object.entries(payload.data)) { const partialLocaleData = { [fileId]: fileData }; + // TODO: data is partial, but meta is full. That's not optimal. + const partialPayload = { data: partialLocaleData, meta: payload.meta }; const partialResult = await this.translator( sourceLocale, targetLocale, - partialLocaleData, - payload.meta, + partialPayload, ); resultData[fileId] = partialResult.data[fileId]; } diff --git a/packages/cli/src/services/translator.ts b/packages/cli/src/services/translator.ts deleted file mode 100644 index e48a17bcc..000000000 --- a/packages/cli/src/services/translator.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { createId } from "@paralleldrive/cuid2"; - -export type TranslatorFn = { - (sourceLocale: string, targetLocale: string, data: Record, meta: any): Promise>; -}; - -export type CreateTranslatorOptions = { - apiUrl: string; - apiKey: string; - skipCache: boolean; - cacheOnly: boolean; -}; - -export function createTranslator(options: CreateTranslatorOptions): TranslatorFn { - return async (sourceLocale, targetLocale, data, meta) => { - const workflowId = createId(); - const res = await fetch(`${options.apiUrl}/i18n`, { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${options.apiKey}`, - }, - body: JSON.stringify({ - params: { - workflowId, - cacheOnly: options.cacheOnly, - skipCache: options.skipCache, - }, - locale: { - source: sourceLocale, - target: targetLocale, - }, - meta, - data, - }, null, 2), - }); - - if (!res.ok) { - if (res.status === 400) { - throw new Error(`Invalid request: ${res.statusText}`); - } else { - const errorText = await res.text(); - throw new Error(errorText); - } - } - - const payload = await res.json(); - - return payload; - }; -} From 15c2e6ee42880b61b6afeaa8795c04560b1fa801 Mon Sep 17 00:00:00 2001 From: maxprilutskiy Date: Thu, 2 May 2024 17:14:49 +0200 Subject: [PATCH 15/24] feat: add markdown bucket processor --- packages/cli/src/services/bucket/markdown.ts | 33 ++++++++++++++++++++ pnpm-lock.yaml | 6 ++++ 2 files changed, 39 insertions(+) create mode 100644 packages/cli/src/services/bucket/markdown.ts diff --git a/packages/cli/src/services/bucket/markdown.ts b/packages/cli/src/services/bucket/markdown.ts new file mode 100644 index 000000000..1250f109e --- /dev/null +++ b/packages/cli/src/services/bucket/markdown.ts @@ -0,0 +1,33 @@ +import { IBucketProcessor } from "./core.js"; +import { BaseBucketProcessor } from "./base.js"; +import _ from 'lodash'; +import objectHash from 'object-hash'; + +export class MarkdownBucketProcessor extends BaseBucketProcessor implements IBucketProcessor { + protected override _validateBucketPath(bucketPath: string): void { + if (!bucketPath.includes('[lang]')) { + throw new Error(`Invalid bucket path: ${bucketPath}. The path must include the [lang] placeholder.`); + } + if (!bucketPath.endsWith('.md')) { + throw new Error(`Invalid bucket path: ${bucketPath}. The path must have a .md file extension.`); + } + } + + protected override _resolveDataFilePath(locale: string): string { + return this.bucketPath.replace('[lang]', locale); + } + + protected override async _deserializeData(content: string): Promise> { + const result = _.chain(content) + .split('\n\n') + .map((content, index) => [objectHash({ content, index }), content]) + .fromPairs() + .value(); + + return result; + } + + protected override async _serializeDataContent(data: Record): Promise { + return _.values(data).join('\n\n'); + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a019b688a..53b102aff 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -210,6 +210,9 @@ importers: lodash: specifier: ^4.17.21 version: 4.17.21 + object-hash: + specifier: ^3.0.0 + version: 3.0.0 open: specifier: ^10.1.0 version: 10.1.0 @@ -244,6 +247,9 @@ importers: '@types/node': specifier: ^20 version: 20.11.30 + '@types/object-hash': + specifier: ^3.0.6 + version: 3.0.6 packages/compiler: dependencies: From e33410931b3c4fca1c16a8e82f8e61a1f022fa28 Mon Sep 17 00:00:00 2001 From: maxprilutskiy Date: Thu, 2 May 2024 17:15:07 +0200 Subject: [PATCH 16/24] feat: add bucket processors --- packages/cli/src/services/bucket/base.ts | 91 ++++++++++++++++ packages/cli/src/services/bucket/core.ts | 8 +- packages/cli/src/services/bucket/json.ts | 25 ++++- packages/cli/src/services/bucket/jsonlike.ts | 64 ----------- packages/cli/src/services/bucket/replexica.ts | 96 +++++----------- packages/cli/src/services/bucket/xcode.ts | 103 ++++++++++++++++++ .../cli/src/services/bucket/yaml-root-key.ts | 15 +++ packages/cli/src/services/bucket/yaml.ts | 28 +++++ 8 files changed, 291 insertions(+), 139 deletions(-) create mode 100644 packages/cli/src/services/bucket/base.ts delete mode 100644 packages/cli/src/services/bucket/jsonlike.ts create mode 100644 packages/cli/src/services/bucket/xcode.ts create mode 100644 packages/cli/src/services/bucket/yaml-root-key.ts create mode 100644 packages/cli/src/services/bucket/yaml.ts diff --git a/packages/cli/src/services/bucket/base.ts b/packages/cli/src/services/bucket/base.ts new file mode 100644 index 000000000..708a676d0 --- /dev/null +++ b/packages/cli/src/services/bucket/base.ts @@ -0,0 +1,91 @@ +import _ from "lodash"; +import fs from 'fs'; +import { BucketPayload, BucketTranslatorFn, IBucketProcessor } from "./core.js"; + +export abstract class BaseBucketProcessor implements IBucketProcessor { + constructor( + protected bucketPath: string, + private translator: BucketTranslatorFn, + ) { + this._validateBucketPath(bucketPath); + } + + async load(locale: string): Promise { + const [ + data, + meta, + ] = await Promise.all([ + this._loadData(locale), + this._loadMeta(), + ]); + + const rawResult = { data, meta }; + const result = await this._postLoad(rawResult, locale); + return result; + } + + async translate(payload: BucketPayload, sourceLocale: string, targetLocale: string): Promise { + // The data contains key-value pairs, so let's translate + // the values in batches of 20 keys max. + const resultData: Record = {}; + + const keys = Object.keys(payload.data); + const batches = _.chunk(keys, 20); + for (const batch of batches) { + const partialData = _.pick(payload.data, batch); + const partialPayload = { data: partialData, meta: payload.meta }; + const partialResult = await this.translator(sourceLocale, targetLocale, partialPayload); + _.merge(resultData, partialResult); + } + + const result = { data: resultData, meta: payload.meta }; + return result; + } + + async save(locale: string, rawPayload: BucketPayload): Promise { + const payload = await this._preSave(rawPayload, locale); + + await this._saveData(locale, payload.data); + + return payload; + } + + protected async _postLoad(payload: BucketPayload, locale: string): Promise { + return payload; + } + + protected async _preSave(payload: BucketPayload, locale: string): Promise { + return payload; + } + + protected async _loadData(locale: string): Promise> { + const filePath = this._resolveDataFilePath(locale); + const exists = await fs.existsSync(filePath); + if (!exists) { return {}; } + + const rawContent = await fs.readFileSync(filePath, 'utf8'); + const data = await this._deserializeData(rawContent); + return data; + } + + protected async _loadMeta(): Promise | null> { + return null; + } + + protected async _saveData(locale: string, data: Record): Promise> { + const filePath = this._resolveDataFilePath(locale); + const content = await this._serializeDataContent(data); + await fs.writeFileSync(filePath, content); + return data; + } + + protected _validateBucketPath(bucketPath: string): void { + // noop + } + + protected abstract _deserializeData(content: string): Promise>; + + protected abstract _serializeDataContent(data: Record): Promise; + + protected abstract _resolveDataFilePath(locale: string): string; +} diff --git a/packages/cli/src/services/bucket/core.ts b/packages/cli/src/services/bucket/core.ts index 0183bf938..1bb802459 100644 --- a/packages/cli/src/services/bucket/core.ts +++ b/packages/cli/src/services/bucket/core.ts @@ -4,6 +4,9 @@ import { ReplexicaBucketProcessor } from './replexica.js'; import { contentTypes, contentTypeSchema } from '@replexica/spec'; import { JsonBucketProcessor } from './json.js'; import { createId } from '@paralleldrive/cuid2'; +import { YamlBucketProcessor } from './yaml.js'; +import { YamlRootKeyBucketProcessor } from './yaml-root-key.js'; +import { XcodeBucketProcessor } from './xcode.js'; // Bucket processor @@ -19,7 +22,7 @@ export type BucketTranslatorFn = { export interface IBucketProcessor { load(locale: string): Promise; translate(payload: BucketPayload, sourceLocale: string, targetLocale: string): Promise; - save(locale: string, payload: BucketPayload): Promise; + save(locale: string, payload: BucketPayload): Promise; } @@ -36,6 +39,9 @@ export function createBucketProcessor( default: throw new Error(`Unknown bucket type: ${bucketType}`); case 'replexica': return new ReplexicaBucketProcessor(bucketPath, translator); case 'json': return new JsonBucketProcessor(bucketPath, translator); + case 'yaml': return new YamlBucketProcessor(bucketPath, translator); + case 'yaml-root-key': return new YamlRootKeyBucketProcessor(bucketPath, translator); + case 'xcode': return new XcodeBucketProcessor(bucketPath, translator); } } diff --git a/packages/cli/src/services/bucket/json.ts b/packages/cli/src/services/bucket/json.ts index 9e505e50b..92a88bd67 100644 --- a/packages/cli/src/services/bucket/json.ts +++ b/packages/cli/src/services/bucket/json.ts @@ -1,6 +1,25 @@ -import { IBucketProcessor } from "./core.js"; -import { JsonLikeBucketProcessor } from "./jsonlike.js"; +import { BucketTranslatorFn, IBucketProcessor } from "./core.js"; +import { BaseBucketProcessor } from "./base.js"; -export class JsonBucketProcessor extends JsonLikeBucketProcessor implements IBucketProcessor { +export class JsonBucketProcessor extends BaseBucketProcessor implements IBucketProcessor { + protected override _validateBucketPath(bucketPath: string): void { + if (!bucketPath.includes('[lang]')) { + throw new Error(`Invalid bucket path: ${bucketPath}. The path must include the [lang] placeholder.`); + } + if (!bucketPath.endsWith('.json')) { + throw new Error(`Invalid bucket path: ${bucketPath}. The path must have a .json file extension.`); + } + } + protected override _resolveDataFilePath(locale: string): string { + return this.bucketPath.replace('[lang]', locale); + } + + protected override async _deserializeData(content: string): Promise> { + return JSON.parse(content); + } + + protected override async _serializeDataContent(data: Record): Promise { + return JSON.stringify(data, null, 2); + } } diff --git a/packages/cli/src/services/bucket/jsonlike.ts b/packages/cli/src/services/bucket/jsonlike.ts deleted file mode 100644 index 2680f2df7..000000000 --- a/packages/cli/src/services/bucket/jsonlike.ts +++ /dev/null @@ -1,64 +0,0 @@ -import _ from "lodash"; -import fs from 'fs'; -import { BucketPayload, BucketTranslatorFn, IBucketProcessor } from "./core.js"; - -export abstract class JsonLikeBucketProcessor implements IBucketProcessor { - constructor( - private bucketPath: string, - private translator: BucketTranslatorFn, - ) { - if (!bucketPath.includes('[lang]')) { - throw new Error(`Invalid bucket path: ${bucketPath}. The path must include the [lang] placeholder.`); - } - if (!bucketPath.endsWith('.json')) { - throw new Error(`Invalid bucket path: ${bucketPath}. The path must have a .json file extension.`); - } - } - - async load(locale: string): Promise { - const filePath = this.bucketPath.replace('[lang]', locale); - const exists = await fs.existsSync(filePath); - if (!exists) { return { data: {}, meta: null }; } - - const rawContent = await fs.readFileSync(filePath, 'utf8'); - const data = JSON.parse(rawContent); - - const rawResult = { data, meta: null }; - const result = await this.postLoad(rawResult, locale); - return result; - } - - async postLoad(payload: BucketPayload, locale: string): Promise { - return payload; - } - - async translate(payload: BucketPayload, sourceLocale: string, targetLocale: string): Promise { - // The data contains key-value pairs, so let's translate - // the values in batches of 20 keys max. - const resultData: Record = {}; - - const keys = Object.keys(payload.data); - const batches = _.chunk(keys, 20); - for (const batch of batches) { - const partialData = _.pick(payload.data, batch); - const partialPayload = { data: partialData, meta: payload.meta }; - const partialResult = await this.translator(sourceLocale, targetLocale, partialPayload); - _.merge(resultData, partialResult); - } - - const result = { data: resultData, meta: payload.meta }; - return result; - } - - async preSave(payload: BucketPayload, locale: string): Promise { - return payload; - } - - async save(locale: string, rawPayload: BucketPayload): Promise { - const payload = await this.preSave(rawPayload, locale); - - const filePath = this.bucketPath.replace('[lang]', locale); - const content = JSON.stringify(payload.data, null, 2); - await fs.writeFileSync(filePath, content); - } -} diff --git a/packages/cli/src/services/bucket/replexica.ts b/packages/cli/src/services/bucket/replexica.ts index 4d45e960a..853ddb4a5 100644 --- a/packages/cli/src/services/bucket/replexica.ts +++ b/packages/cli/src/services/bucket/replexica.ts @@ -1,60 +1,28 @@ import path from "path"; import fs from "fs"; -import { IBucketProcessor, BucketTranslatorFn, BucketPayload } from "./core.js"; +import { IBucketProcessor, BucketPayload } from "./core.js"; +import { BaseBucketProcessor } from "./base.js"; -export class ReplexicaBucketProcessor implements IBucketProcessor { - constructor( - private bucketPath: string, - private translator: BucketTranslatorFn, - ) { +export class ReplexicaBucketProcessor extends BaseBucketProcessor implements IBucketProcessor { + protected override _validateBucketPath(bucketPath: string): void { if (bucketPath !== '') { throw new Error(`Unknown bucket path: ${bucketPath}. Replexica bucket path must be an empty string: ''.`); } } - async load(locale: string): Promise { - const [ - meta, - data, - ] = await Promise.all([ - this._loadMeta(), - this._loadData(locale), - ]); - - return { data, meta }; + protected _resolveDataFilePath(locale: string): string { + return path.resolve(process.cwd(), 'node_modules', '@replexica/translations', `${locale}.json`); } - async translate(payload: BucketPayload, sourceLocale: string, targetLocale: string): Promise { - const resultData: any = {}; - // Currently the split is done by fileId, but as files can - // get quite large, we might want to split by a certain number - // of files' scopes instead. - for (const [fileId, fileData] of Object.entries(payload.data)) { - const partialLocaleData = { [fileId]: fileData }; - // TODO: data is partial, but meta is full. That's not optimal. - const partialPayload = { data: partialLocaleData, meta: payload.meta }; - const partialResult = await this.translator( - sourceLocale, - targetLocale, - partialPayload, - ); - resultData[fileId] = partialResult.data[fileId]; - } - - return { - data: resultData, - meta: payload.meta, - }; + protected _deserializeData(content: string): Promise> { + return Promise.resolve(JSON.parse(content)); } - async save(locale: string, payload: BucketPayload): Promise { - await Promise.all([ - this._saveFullData(locale, payload), - this._saveClientData(locale, payload), - ]); + protected _serializeDataContent(data: Record): Promise { + return Promise.resolve(JSON.stringify(data, null, 2)); } - private async _loadMeta(): Promise { + protected override async _loadMeta(): Promise { const bucketDir = path.resolve(process.cwd(), 'node_modules', '@replexica/translations'); const metaFilePath = path.resolve(bucketDir, '.replexica.json'); @@ -68,36 +36,20 @@ export class ReplexicaBucketProcessor implements IBucketProcessor { return meta; } - private async _loadData(locale: string): Promise { - const bucketDir = path.resolve(process.cwd(), 'node_modules', '@replexica/translations'); - const dataFilePath = path.resolve(bucketDir, `${locale}.json`); - - const exists = await fs.existsSync(dataFilePath); - if (!exists) { return {}; } - - const rawData = await fs.readFileSync(dataFilePath, 'utf8'); - const data = JSON.parse(rawData); - - return data; + protected override async _saveData(locale: string, data: Record): Promise> { + const savedFullData = await super._saveData(locale, data); // Save full data + await this._saveClientData(locale, savedFullData); // Save client data + return savedFullData; } - private async _saveFullData(locale: string, payload: BucketPayload): Promise { - const bucketDir = path.resolve(process.cwd(), 'node_modules', '@replexica/translations'); - const dataFilePath = path.resolve(bucketDir, `${locale}.json`); - - const content = JSON.stringify(payload.data, null, 2); - await fs.writeFileSync(dataFilePath, content); - } + private async _saveClientData(locale: string, data: Record): Promise> { + const fullDataFilePath = this._resolveDataFilePath(locale); + const clientDataFilePath = fullDataFilePath.replace('.json', '.client.json'); - private async _saveClientData(locale: string, payload: BucketPayload): Promise { - const bucketDir = path.resolve(process.cwd(), 'node_modules', '@replexica/translations'); - const dataFilePath = path.resolve(bucketDir, `${locale}.client.json`); - - const newData = { - ...payload.data, - }; + const meta = await this._loadMeta(); + const newData = { ...data }; - for (const [fileId, fileData] of Object.entries(payload.meta.files || {})) { + for (const [fileId, fileData] of Object.entries(meta.files || {})) { const isClient = (fileData as any).isClient; if (!isClient) { @@ -105,7 +57,9 @@ export class ReplexicaBucketProcessor implements IBucketProcessor { } } - const content = JSON.stringify(newData, null, 2); - await fs.writeFileSync(dataFilePath, content); + const content = await this._serializeDataContent(newData); + await fs.writeFileSync(clientDataFilePath, content); + + return newData; } } \ No newline at end of file diff --git a/packages/cli/src/services/bucket/xcode.ts b/packages/cli/src/services/bucket/xcode.ts new file mode 100644 index 000000000..b613e1df6 --- /dev/null +++ b/packages/cli/src/services/bucket/xcode.ts @@ -0,0 +1,103 @@ +import { BucketPayload, IBucketProcessor } from "./core.js"; +import { BaseBucketProcessor } from "./base.js"; + +export class XcodeBucketProcessor extends BaseBucketProcessor implements IBucketProcessor { + protected override _validateBucketPath(bucketPath: string): void { + if (!bucketPath.endsWith('.xcstrings')) { + throw new Error(`Unknown bucket path: ${bucketPath}. Xcode bucket path must end with '.xcstrings'.`); + } + } + + protected override _resolveDataFilePath(): string { + return this.bucketPath; + } + + protected override async _deserializeData(content: string): Promise> { + const parsed = JSON.parse(content); + return parsed; + } + + protected override async _serializeDataContent(data: Record): Promise { + return JSON.stringify(data, null, 2); + } + + protected async _postLoad(payload: BucketPayload, locale: string): Promise { + const parsed = payload.data; + + const resultData: Record = {}; + + for (const [translationKey, _translationEntity] of Object.entries(parsed.strings)) { + const rootTranslationEntity = _translationEntity as any; + const langTranslationEntity = rootTranslationEntity?.localizations?.[locale]; + if (langTranslationEntity) { + if ('stringUnit' in langTranslationEntity) { + resultData[translationKey] = langTranslationEntity.stringUnit.value; + } else if ('variations' in langTranslationEntity) { + if ('plural' in langTranslationEntity.variations) { + resultData[translationKey] = { + one: langTranslationEntity.variations.plural.one?.stringUnit?.value || '', + other: langTranslationEntity.variations.plural.other?.stringUnit?.value || '', + zero: langTranslationEntity.variations.plural.zero?.stringUnit?.value || '', + }; + } + } + } + } + + const result: BucketPayload = { ...payload, data: resultData }; + return result; + } + + protected async _preSave(payload: BucketPayload, locale: string): Promise { + const langDataToMerge: any = {}; + langDataToMerge.strings = {}; + + for (const [key, value] of Object.entries(payload.data)) { + if (typeof value === 'string') { + langDataToMerge.strings[key] = { + extractionState: 'manual', + localizations: { + [locale]: { + stringUnit: { + state: 'translated', + value, + }, + }, + }, + }; + } else { + langDataToMerge.strings[key] = { + extractionState: 'manual', + localizations: { + [locale]: { + variations: { + plural: { + one: { + stringUnit: { + state: 'translated', + value: value.one, + }, + }, + other: { + stringUnit: { + state: 'translated', + value: value.other, + }, + }, + zero: { + stringUnit: { + state: 'translated', + value: value.zero, + }, + }, + }, + }, + }, + }, + }; + } + } + + return langDataToMerge; + } +} diff --git a/packages/cli/src/services/bucket/yaml-root-key.ts b/packages/cli/src/services/bucket/yaml-root-key.ts new file mode 100644 index 000000000..f9dc45908 --- /dev/null +++ b/packages/cli/src/services/bucket/yaml-root-key.ts @@ -0,0 +1,15 @@ +import { IBucketProcessor } from "./core.js"; +import { YamlBucketProcessor } from './yaml.js'; + +export class YamlRootKeyBucketProcessor extends YamlBucketProcessor implements IBucketProcessor { + protected override async _loadData(locale: string): Promise> { + const data = await super._loadData(locale); + return data[locale]; + } + + protected override async _saveData(locale: string, data: Record): Promise> { + const finalData = { [locale]: data }; + const result = await super._saveData(locale, finalData); + return result; + } +} diff --git a/packages/cli/src/services/bucket/yaml.ts b/packages/cli/src/services/bucket/yaml.ts new file mode 100644 index 000000000..c523f35bc --- /dev/null +++ b/packages/cli/src/services/bucket/yaml.ts @@ -0,0 +1,28 @@ +import YAML from 'yaml'; +import { IBucketProcessor } from "./core.js"; +import { BaseBucketProcessor } from "./base.js"; + +export class YamlBucketProcessor extends BaseBucketProcessor implements IBucketProcessor { + protected override _validateBucketPath(bucketPath: string): void { + if (!bucketPath.includes('[lang]')) { + throw new Error(`Invalid bucket path: ${bucketPath}. The path must include the [lang] placeholder.`); + } + + const supportedExtensions = ['.yaml', '.yml']; + if (!supportedExtensions.some((ext) => bucketPath.endsWith(ext))) { + throw new Error(`Invalid bucket path: ${bucketPath}. The path must have a ${supportedExtensions.join(' or ')} file extension.`); + } + } + + protected override _resolveDataFilePath(locale: string): string { + return this.bucketPath.replace('[lang]', locale); + } + + protected _deserializeData(content: string): Promise> { + return YAML.parse(content); + } + + protected _serializeDataContent(data: Record): Promise { + return Promise.resolve(YAML.stringify(data)); + } +} From 9add3c4ba430bb2953bdf47604eb664b5e0e0172 Mon Sep 17 00:00:00 2001 From: maxprilutskiy Date: Thu, 2 May 2024 17:15:33 +0200 Subject: [PATCH 17/24] refactor: remove old cli code --- packages/cli/package.json | 6 +- packages/cli/src/index.ts | 2 - packages/cli/src/localize/config/default.ts | 58 ----- packages/cli/src/localize/config/load.ts | 14 -- packages/cli/src/localize/config/save.ts | 10 - packages/cli/src/localize/config/schema.ts | 21 -- packages/cli/src/localize/engine/client.ts | 78 ------- packages/cli/src/localize/index.ts | 211 ------------------ .../localize/lib/lang-data-processor/base.ts | 44 ---- .../localize/lib/lang-data-processor/json.ts | 39 ---- .../lib/lang-data-processor/markdown.ts | 36 --- .../localize/lib/lang-data-processor/xcode.ts | 124 ---------- .../lib/lang-data-processor/yaml-ror.ts | 15 -- .../localize/lib/lang-data-processor/yaml.ts | 40 ---- 14 files changed, 4 insertions(+), 694 deletions(-) delete mode 100644 packages/cli/src/localize/config/default.ts delete mode 100644 packages/cli/src/localize/config/load.ts delete mode 100644 packages/cli/src/localize/config/save.ts delete mode 100644 packages/cli/src/localize/config/schema.ts delete mode 100644 packages/cli/src/localize/engine/client.ts delete mode 100644 packages/cli/src/localize/index.ts delete mode 100644 packages/cli/src/localize/lib/lang-data-processor/base.ts delete mode 100644 packages/cli/src/localize/lib/lang-data-processor/json.ts delete mode 100644 packages/cli/src/localize/lib/lang-data-processor/markdown.ts delete mode 100644 packages/cli/src/localize/lib/lang-data-processor/xcode.ts delete mode 100644 packages/cli/src/localize/lib/lang-data-processor/yaml-ror.ts delete mode 100644 packages/cli/src/localize/lib/lang-data-processor/yaml.ts diff --git a/packages/cli/package.json b/packages/cli/package.json index 5c7125caf..3f600ceda 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -23,9 +23,9 @@ "author": "", "license": "ISC", "dependencies": { - "@replexica/spec": "workspace:*", "@inquirer/prompts": "^4.3.1", "@paralleldrive/cuid2": "^2.2.2", + "@replexica/spec": "workspace:*", "commander": "^12.0.0", "cors": "^2.8.5", "dotenv": "^16.4.5", @@ -33,6 +33,7 @@ "flat": "^6.0.1", "ini": "^4.1.2", "lodash": "^4.17.21", + "object-hash": "^3.0.0", "open": "^10.1.0", "ora": "^8.0.1", "typescript": "^5.4.5", @@ -45,6 +46,7 @@ "@types/express": "^4.17.21", "@types/ini": "^4.1.0", "@types/lodash": "^4.17.0", - "@types/node": "^20" + "@types/node": "^20", + "@types/object-hash": "^3.0.6" } } \ No newline at end of file diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 9863726a6..98613c7ed 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -3,7 +3,6 @@ import { Command } from 'commander'; import i18nCmd from './i18n.js'; import authCmd from './auth.js'; import initCmd from './init.js'; -import localizeCmd from './localize/index.js'; export default new Command() .name('replexica') @@ -12,5 +11,4 @@ export default new Command() .addCommand(i18nCmd) .addCommand(authCmd) .addCommand(initCmd) - .addCommand(localizeCmd) // legacy .parse(process.argv); diff --git a/packages/cli/src/localize/config/default.ts b/packages/cli/src/localize/config/default.ts deleted file mode 100644 index 80b97a03c..000000000 --- a/packages/cli/src/localize/config/default.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { ConfigSchema } from "./schema.js"; - -const DEMO_DIRECTORY_NAME = 'replexica-demo'; -export const defaultPaths = { - config: '.replexica/config.yml', - dictionaryEn: `${DEMO_DIRECTORY_NAME}/en.json`, - dictionaryPattern: `${DEMO_DIRECTORY_NAME}/[lang].json`, - githubWorkflow: '.github/workflows/replexica.yml', -}; - -export const defaultConfig: ConfigSchema = { - version: 1, - languages: { source: 'en', target: ['es'] }, - projects: [ - { name: 'demo', type: 'json', path: defaultPaths.dictionaryPattern }, - ], -}; - - -export const defaultGithubWorkflow = { - name: 'Replexica', - on: { - workflow_dispatch: null, - push: { branches: ['main'] }, - pull_request: { branches: ['main'] }, - }, - permissions: { - contents: 'write', - 'pull-requests': 'write', - checks: 'write', - actions: 'write', - issues: 'write', - }, - jobs: { - localize: { - 'runs-on': 'ubuntu-latest', - 'steps': [ - { - name: 'Checkout', - uses: 'actions/checkout@v2', - }, - { - name: 'Replexica', - uses: 'replexica/github-action@main', - env: { - REPLEXICA_API_KEY: '${{ secrets.REPLEXICA_API_KEY }}', - GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}', - }, - }, - ], - }, - }, -}; - -export const demoEnDictionary = { - 'home.title': 'Replexica', - 'home.description': 'Replexica is an AI-powered localization-as-a-service platform for modern SaaS.', -}; diff --git a/packages/cli/src/localize/config/load.ts b/packages/cli/src/localize/config/load.ts deleted file mode 100644 index 619887fd1..000000000 --- a/packages/cli/src/localize/config/load.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { readFileSync, existsSync } from 'fs'; -import {parse} from 'yaml'; -import { configSchema } from './schema.js'; - -export function loadConfig(configFilePath: string) { - const configExists = existsSync(configFilePath); - if (!configExists) { return null; } - - const configFileContent = readFileSync(configFilePath, 'utf8'); - const configObject = parse(configFileContent); - const config = configSchema.parse(configObject); - - return config; -} diff --git a/packages/cli/src/localize/config/save.ts b/packages/cli/src/localize/config/save.ts deleted file mode 100644 index eaee41f9f..000000000 --- a/packages/cli/src/localize/config/save.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { writeFileSync, mkdirSync } from 'fs'; -import { dirname } from 'path'; -import { stringify } from 'yaml'; -import { ConfigSchema } from './schema.js'; - -export function saveConfig(configFilePath: string, configObject: ConfigSchema) { - const configYaml = stringify(configObject); - mkdirSync(dirname(configFilePath), { recursive: true }); - writeFileSync(configFilePath, configYaml); -} diff --git a/packages/cli/src/localize/config/schema.ts b/packages/cli/src/localize/config/schema.ts deleted file mode 100644 index f32257c18..000000000 --- a/packages/cli/src/localize/config/schema.ts +++ /dev/null @@ -1,21 +0,0 @@ -import Z from 'zod'; -import { sourceLocaleSchema, targetLocaleSchema, contentTypeSchema } from '@replexica/spec'; - -const languageSchema = Z.object({ - source: sourceLocaleSchema, - target: Z.array(targetLocaleSchema), -}); - -const contentItemSchema = Z.object({ - name: Z.string(), - type: contentTypeSchema.optional().default('json'), - path: Z.string(), -}); - -export const configSchema = Z.object({ - version: Z.literal(1), - languages: languageSchema, - projects: Z.array(contentItemSchema).default([]).optional(), -}); - -export type ConfigSchema = Z.infer; diff --git a/packages/cli/src/localize/engine/client.ts b/packages/cli/src/localize/engine/client.ts deleted file mode 100644 index 2e6789b3c..000000000 --- a/packages/cli/src/localize/engine/client.ts +++ /dev/null @@ -1,78 +0,0 @@ -export function getReplexicaClient() { - const { REPLEXICA_API_KEY, REPLEXICA_HOST } = process.env; - if (!REPLEXICA_API_KEY) { - throw new Error('REPLEXICA_API_KEY is required'); - } - - return new Replexica({ - apiKey: REPLEXICA_API_KEY, - host: REPLEXICA_HOST, - }); -} - -export class Replexica { - constructor( - private params: ReplexicaInitParams, - ) {} - - async extractLocalizableText(fileContent: string, relativePath: string): Promise { - const response = await this.exec('/extract', { - relativePath, - content: fileContent, - }); - const data = response; - return data; - } - - async localizeJson(params: LocalizeParams>): Promise>> { - const response = await this.exec('/localize/json', params); - const data = response; - return data; - } - - private async exec(path: string, payload: any) { - const host = this.params.host || 'https://engine.replexica.com'; - const url = new URL(path, host); - const response = await fetch(url.href, { - method: 'POST', - headers: { - Authorization: `Bearer ${this.params.apiKey}`, - }, - body: JSON.stringify(payload), - }); - if (!response.ok) { - const message = await response.text(); - throw new Error(message); - } - - const data = await response.json(); - return data; - } -} - -export type ReplexicaInitParams = { - apiKey: string; - host?: string; -}; - -export type LocalizeParams = { - groupId: string; - - triggerType: string; - triggerName: string; - - sourceLocale: string; - targetLocale: string; - data: T; -}; - -export type LocalizeResult = { - sourceLocale: string; - targetLocale: string; - data: T; -} - -export type ExtractResult = { - dictionary: Record; - content: string; -}; diff --git a/packages/cli/src/localize/index.ts b/packages/cli/src/localize/index.ts deleted file mode 100644 index c7c116cc4..000000000 --- a/packages/cli/src/localize/index.ts +++ /dev/null @@ -1,211 +0,0 @@ -import path from 'path'; -import fs from 'fs/promises'; -import { loadConfig } from './config/load.js'; -import { ConfigSchema } from './config/schema.js'; -import _ from 'lodash'; -import { getReplexicaClient } from './engine/client.js'; -import { createId } from '@paralleldrive/cuid2'; -import YAML from 'yaml'; -import Crypto from 'crypto'; -import { YamlRorLangDataProcessor } from './lib/lang-data-processor/yaml-ror.js'; -import { MarkdownLangDataProcessor } from './lib/lang-data-processor/markdown.js'; -import { JsonLangDataProcessor } from './lib/lang-data-processor/json.js'; -import { YamlLangDataProcessor } from './lib/lang-data-processor/yaml.js'; -import { XcodeLangDataProcessor } from './lib/lang-data-processor/xcode.js'; -import { ILangDataProcessor, LangDataType } from './lib/lang-data-processor/base.js'; -import { Command } from 'commander'; - -const TRANSLATIONS_PER_BATCH = 25; - -const langDataProcessorsMap = new Map() - .set('markdown', new MarkdownLangDataProcessor()) - .set('json', new JsonLangDataProcessor()) - .set('yaml', new YamlLangDataProcessor()) - .set('yaml-root-key', new YamlRorLangDataProcessor()) - .set('xcode', new XcodeLangDataProcessor()); - -export default new Command() - .command('localize') - .description('Localize i18n JSON with Replexica') - .helpOption('-h, --help', 'Show help') - .argument('[root]', 'Root directory of the repository, containing the .replexica/config.yml config file.', '.') - .option('--trigger-type ', 'Environment from which the localization is triggered', 'cli') - .option('--trigger-name ', 'Name of the trigger', '') -// .action(async (root, options) => { -// const config = await extractConfig(root, options.triggerType, options.triggerName); -// for (const project of config.projects) { -// const sourceLangData = await loadProjectLangData(project, config.sourceLang); -// const changedKeys = await calculateChangedKeys(project.name, sourceLangData); - -// // Write hash file at the beginning of the process -// // So that if it fails in the middle, we won't have -// // to re-translate everything from scratch again -// await writeHashFile(project.name, sourceLangData); - -// for (const targetLang of config.targetLangs) { -// const targetLangData = await loadProjectLangData(project, targetLang); - -// const removedKeys = _.difference(Object.keys(targetLangData), Object.keys(sourceLangData)); -// const missingKeys = _.difference(Object.keys(sourceLangData), Object.keys(targetLangData)); - -// const projectLogPrefix = `[${project.name}]`; -// console.info(`${projectLogPrefix} Removed: ${removedKeys.length}. Changed: ${changedKeys.length}. Missing: ${missingKeys.length}.`); - -// const keysToTranslate = _.uniq([...changedKeys, ...missingKeys]); - -// const translationLogPrefix = `${projectLogPrefix} (${config.sourceLang} -> ${targetLang})`; -// console.log(`${translationLogPrefix} Translating ${keysToTranslate.length} keys`); - -// let langDataUpdate: Record = {}; -// if (keysToTranslate.length) { -// const keysToTranslateChunks = _.chunk(keysToTranslate, TRANSLATIONS_PER_BATCH); - -// let translatedKeysCount = 0; -// const groupId = `leg_${createId()}`; -// for (const keysToTranslateChunk of keysToTranslateChunks) { -// console.log(`${translationLogPrefix} Translating keys, ${translatedKeysCount}/${keysToTranslate.length}`); -// const partialDiffRecord = _.pick(sourceLangData, keysToTranslateChunk); -// const partialLangDataUpdate = await translateRecord( -// config.sourceLang, -// targetLang, -// partialDiffRecord, -// groupId, -// ); -// langDataUpdate = _.merge(langDataUpdate, partialLangDataUpdate); - -// translatedKeysCount += keysToTranslateChunk.length; -// } - -// console.log(`Done`); -// } else { -// console.log(`Skipped`); -// } - -// const newTargetLangData = _.chain(targetLangData) -// .merge(langDataUpdate) -// .omit(removedKeys) -// .value(); - -// await saveProjectLangData(project, targetLang, newTargetLangData); -// } -// } -// }); -// async function writeHashFile(projectName: string, sourceLangData: Record) { -// const projectHashfileNode: Record = {}; -// for (const [key, value] of Object.entries(sourceLangData)) { -// const valueHash = Crypto -// .createHash('sha256') -// .update(value) -// .digest('hex'); - -// projectHashfileNode[key] = valueHash; -// } -// const replexicaHashfileContent = await fs.readFile('.replexica/hash.yaml', 'utf-8') -// .catch(() => '') -// .then((content) => content.trim() || ''); -// const replexicaHashfile = YAML.parse(replexicaHashfileContent) || {} as Record; -// replexicaHashfile.version = replexicaHashfile.version || 1; -// replexicaHashfile[projectName] = projectHashfileNode; - -// const newReplexicaHashfileContent = [ -// '# DO NOT MODIFY THIS FILE MANUALLY', -// '# This file is auto-generated by Replexica. Please keep it in your version control system.', -// YAML.stringify(replexicaHashfile), -// ].join('\n'); -// await fs.writeFile('.replexica/hash.yaml', newReplexicaHashfileContent); -// } - -// async function calculateChangedKeys(projectName: string, sourceLangData: Record): Promise { -// const replexicaHashfileContent = await fs.readFile('.replexica/hash.yaml', 'utf-8') -// .catch(() => '') -// .then((content) => content.trim() || ''); -// const replexicaHashfile = YAML.parse(replexicaHashfileContent) || {} as Record; -// const projectHashfileNode = replexicaHashfile[projectName] || {}; - -// const result: string[] = []; -// for (const [key, value] of Object.entries(sourceLangData)) { -// const valueHash = Crypto -// .createHash('sha256') -// .update(value) -// .digest('hex'); - -// if (projectHashfileNode[key] !== valueHash) { -// result.push(key); -// } -// } - -// return result; -// } - -// async function loadProjectLangData(project: ConfigSchema['projects'][0], lang: string): Promise> { -// const processor = langDataProcessorsMap.get(project.type); -// if (!processor) { throw new Error('Unsupported project type ' + project.type); } - -// const result = await processor.loadLangJson(project.dictionary, lang); - -// return result; -// } - -// async function saveProjectLangData(project: ConfigSchema['projects'][0], lang: string, data: Record) { -// const processor = langDataProcessorsMap.get(project.type); -// if (!processor) { throw new Error('Unsupported project type ' + project.type); } - -// await processor.saveLangJson(project.dictionary, lang, data); -// } - -// async function extractConfig(root: string, triggerType: string, triggerName: string) { -// const configRoot = path.resolve(process.cwd(), root); -// const configFilePath = path.join(configRoot, '.replexica/config.yml'); - -// const configFileExists = await fs.stat(configFilePath).then(() => true).catch(() => false); -// if (!configFileExists) { -// throw new Error(`Config file not found at ${configFilePath}.`); -// } - -// const config: ConfigSchema | null = configFileExists ? loadConfig(configFilePath) : null; - -// const sourceLang = config?.languages.source; -// if (!sourceLang) { -// throw new Error('Source language must be specified.'); -// } - -// const targetLangs = (config?.languages.target || []).filter(Boolean); -// if (targetLangs.length === 0) { -// throw new Error('At least one target language must be specified.'); -// } - -// const projects = config?.projects || []; -// if (projects.length === 0) { -// throw new Error('At least one project must be specified.'); -// } - -// return { -// sourceLang, -// targetLangs, -// projects, -// triggerType, -// triggerName, -// }; -// } - -// async function translateRecord( -// sourceLang: string, -// targetLang: string, -// data: Record, -// groupId: string, -// ): Promise> { -// if (Object.keys(data).length === 0) { return {}; } - -// const replexica = getReplexicaClient(); -// const translateRecordResponse = await replexica.localizeJson({ -// groupId, -// triggerType: 'cli', -// triggerName: 'cli', - -// sourceLocale: sourceLang, -// targetLocale: targetLang, -// data, -// }); - -// return translateRecordResponse.data; -// } diff --git a/packages/cli/src/localize/lib/lang-data-processor/base.ts b/packages/cli/src/localize/lib/lang-data-processor/base.ts deleted file mode 100644 index 076b1e131..000000000 --- a/packages/cli/src/localize/lib/lang-data-processor/base.ts +++ /dev/null @@ -1,44 +0,0 @@ -export type LangDataType = 'json' | 'xcode' | 'yaml' | 'yaml-root-key' | 'markdown'; - -export type LangDataNode = { - [key: string]: string | LangDataNode; -}; - -export interface ILangDataProcessor { - loadLangJson(path: string, lang: string): Promise>; - saveLangJson(path: string, lang: string, langData: Record): Promise; -} - -export abstract class BaseLangDataProcessor { - protected abstract validatePath(path: string, lang: string): Promise; - - protected async preflatten(langData: LangDataNode, lang: string): Promise { - return langData; - } - - protected async flatten(langData: LangDataNode, lang: string): Promise> { - const flat = await import('flat'); - const preparedLangData = await this.preflatten(langData, lang); - const result = flat.flatten>(preparedLangData, { - delimiter: '/', - transformKey: (key) => encodeURIComponent(key), - }); - - return result; - } - - protected async postunflatten(record: LangDataNode, lang: string): Promise { - return record; - } - - protected async unflatten(record: Record, lang: string): Promise { - const flat = await import('flat'); - const rawResult = flat.unflatten, LangDataNode>(record, { - delimiter: '/', - transformKey: (key) => decodeURIComponent(key), - }); - const result = await this.postunflatten(rawResult, lang); - - return result; - } -} \ No newline at end of file diff --git a/packages/cli/src/localize/lib/lang-data-processor/json.ts b/packages/cli/src/localize/lib/lang-data-processor/json.ts deleted file mode 100644 index b8857bc8d..000000000 --- a/packages/cli/src/localize/lib/lang-data-processor/json.ts +++ /dev/null @@ -1,39 +0,0 @@ -import fs from 'fs/promises'; -import path from "path"; -import { BaseLangDataProcessor, ILangDataProcessor, LangDataNode } from "./base.js"; - -export class JsonLangDataProcessor extends BaseLangDataProcessor implements ILangDataProcessor { - override async validatePath(filePathPattern: string): Promise { - if (!filePathPattern.includes('[lang]')) { throw new Error('The file path must include the [lang] placeholder'); } - if (!filePathPattern.endsWith('.json')) { throw new Error('Json dictionary must have .json file extension'); } - } - - async loadLangJson(filePathPattern: string, lang: string): Promise> { - await this.validatePath(filePathPattern); - - const filePath = filePathPattern.replace('[lang]', lang); - const fileExists = await fs.stat(filePath).then(() => true).catch(() => false); - if (!fileExists) { - return {}; - } else { - const fileContent = await fs.readFile(filePath, 'utf8'); - const langData = JSON.parse(fileContent) as LangDataNode; - const result = await this.flatten(langData, lang); - return result; - } - } - - async saveLangJson(filePathPattern: string, lang: string, record: Record): Promise { - await this.validatePath(filePathPattern); - - const langData = await this.unflatten(record, lang); - - const filePath = filePathPattern.replace('[lang]', lang); - const fileContent = JSON.stringify(langData, null, 2); - // Create all directories in the path if they don't exist - const dirPath = path.dirname(filePath); - await fs.mkdir(dirPath, { recursive: true }); - // Write the file - await fs.writeFile(filePath, fileContent); - } -} diff --git a/packages/cli/src/localize/lib/lang-data-processor/markdown.ts b/packages/cli/src/localize/lib/lang-data-processor/markdown.ts deleted file mode 100644 index 290c4e74c..000000000 --- a/packages/cli/src/localize/lib/lang-data-processor/markdown.ts +++ /dev/null @@ -1,36 +0,0 @@ -import fs from 'fs/promises'; -import path from "path"; -import { BaseLangDataProcessor, ILangDataProcessor } from "./base.js"; - -export class MarkdownLangDataProcessor extends BaseLangDataProcessor implements ILangDataProcessor { - override async validatePath(filePathPattern: string): Promise { - if (!filePathPattern.includes('[lang]')) { throw new Error('The file path must include the [lang] placeholder'); } - if (!['.md', '.mdx'].some((ext) => filePathPattern.endsWith(ext))) { throw new Error('Markdown dictionary must have .md or .mdx file extension'); } - } - - async loadLangJson(filePathPattern: string, lang: string): Promise> { - await this.validatePath(filePathPattern); - - const filePath = filePathPattern.replace('[lang]', lang); - const fileExists = await fs.stat(filePath).then(() => true).catch(() => false); - if (!fileExists) { - return {}; - } else { - const fileContent = await fs.readFile(filePath, 'utf8'); - const result = { '': fileContent }; - return result; - } - } - - async saveLangJson(filePathPattern: string, lang: string, record: Record): Promise { - await this.validatePath(filePathPattern); - - const filePath = filePathPattern.replace('[lang]', lang); - const fileContent = record['']; - // Create all directories in the path if they don't exist - const dirPath = path.dirname(filePath); - await fs.mkdir(dirPath, { recursive: true }); - // Write the file - await fs.writeFile(filePath, fileContent); - } -} diff --git a/packages/cli/src/localize/lib/lang-data-processor/xcode.ts b/packages/cli/src/localize/lib/lang-data-processor/xcode.ts deleted file mode 100644 index b480009ff..000000000 --- a/packages/cli/src/localize/lib/lang-data-processor/xcode.ts +++ /dev/null @@ -1,124 +0,0 @@ -import fs from 'fs/promises'; -import _ from "lodash"; -import { BaseLangDataProcessor, ILangDataProcessor, LangDataNode } from "./base.js"; - -export class XcodeLangDataProcessor extends BaseLangDataProcessor implements ILangDataProcessor { - override async validatePath(path: string): Promise { - if (!path.endsWith('.xcstrings')) { throw new Error('Xcode dictionary must have .xcstrings file extension'); } - } - - async loadLangJson(filePath: string, lang: string): Promise> { - await this.validatePath(filePath); - - const fileExists = fs.stat(filePath).then(() => true).catch(() => false); - if (!fileExists) { - return {}; - } else { - const fileContent = await fs.readFile(filePath, 'utf8'); - const langData = await this.parseLangData(fileContent, lang); - const result = await this.flatten(langData, lang); - return result; - } - } - - async saveLangJson(filePath: string, lang: string, record: Record): Promise { - await this.validatePath(filePath); - - const fileExists = await fs.stat(filePath).then(() => true).catch(() => false); - if (!fileExists) { throw new Error('Xcode translation was not found.'); } - - const fileContent = await fs.readFile(filePath, 'utf8'); - const parsed = JSON.parse(fileContent); - - const langData = await this.unflatten(record, lang); - const langDataToMerge = await this.serializeLangDataPartial(langData, lang); - - const result = _.mergeWith(parsed, langDataToMerge, (objValue, srcValue) => { - if (_.isObject(objValue)) { - // If the value is an object, merge it deeply - return _.merge(objValue, srcValue); - } - }); - - // Write the file - await fs.writeFile(filePath, JSON.stringify(result, null, 2)); - } - - private async parseLangData(fileContent: string, lang: string): Promise { - const parsed = JSON.parse(fileContent); - - const result: LangDataNode = {}; - for (const [translationKey, _translationEntity] of Object.entries(parsed.strings)) { - const rootTranslationEntity = _translationEntity as any; - const langTranslationEntity = rootTranslationEntity?.localizations?.[lang]; - if (langTranslationEntity) { - if ('stringUnit' in langTranslationEntity) { - result[translationKey] = langTranslationEntity.stringUnit.value; - } else if ('variations' in langTranslationEntity) { - if ('plural' in langTranslationEntity.variations) { - result[translationKey] = { - one: langTranslationEntity.variations.plural.one?.stringUnit?.value || '', - other: langTranslationEntity.variations.plural.other?.stringUnit?.value || '', - zero: langTranslationEntity.variations.plural.zero?.stringUnit?.value || '', - }; - } - } - } - } - - return result; - } - - private async serializeLangDataPartial(langData: LangDataNode, lang: string): Promise { - const langDataToMerge: any = {}; - langDataToMerge.strings = {}; - - for (const [key, value] of Object.entries(langData)) { - if (typeof value === 'string') { - langDataToMerge.strings[key] = { - extractionState: 'manual', - localizations: { - [lang]: { - stringUnit: { - state: 'translated', - value, - }, - }, - }, - }; - } else { - langDataToMerge.strings[key] = { - extractionState: 'manual', - localizations: { - [lang]: { - variations: { - plural: { - one: { - stringUnit: { - state: 'translated', - value: value.one, - }, - }, - other: { - stringUnit: { - state: 'translated', - value: value.other, - }, - }, - zero: { - stringUnit: { - state: 'translated', - value: value.zero, - }, - }, - }, - }, - }, - }, - }; - } - } - - return langDataToMerge; - } -} diff --git a/packages/cli/src/localize/lib/lang-data-processor/yaml-ror.ts b/packages/cli/src/localize/lib/lang-data-processor/yaml-ror.ts deleted file mode 100644 index 1b306b049..000000000 --- a/packages/cli/src/localize/lib/lang-data-processor/yaml-ror.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { LangDataNode } from './base.js'; -import { YamlLangDataProcessor } from './yaml.js'; - -export class YamlRorLangDataProcessor extends YamlLangDataProcessor { - protected async preflatten(langData: LangDataNode, lang: string): Promise { - const innerNode = langData[lang]; - const result = typeof innerNode === 'object' ? innerNode : {}; - return result; - } - - protected async postunflatten(record: LangDataNode, lang: string): Promise { - const result = { [lang]: record }; - return result; - } -} diff --git a/packages/cli/src/localize/lib/lang-data-processor/yaml.ts b/packages/cli/src/localize/lib/lang-data-processor/yaml.ts deleted file mode 100644 index d4f94e28b..000000000 --- a/packages/cli/src/localize/lib/lang-data-processor/yaml.ts +++ /dev/null @@ -1,40 +0,0 @@ -import fs from 'fs/promises'; -import path from "path"; -import { BaseLangDataProcessor, ILangDataProcessor, LangDataNode } from "./base.js"; -import YAML from 'yaml'; - -export class YamlLangDataProcessor extends BaseLangDataProcessor implements ILangDataProcessor { - override async validatePath(filePathPattern: string): Promise { - if (!filePathPattern.includes('[lang]')) { throw new Error('The file path must include the [lang] placeholder'); } - if (!['.yaml', '.yml'].some((ext) => filePathPattern.endsWith(ext))) { throw new Error('Yaml dictionary must have .yaml or .yml file extension'); } - } - - async loadLangJson(filePathPattern: string, lang: string): Promise> { - await this.validatePath(filePathPattern); - - const filePath = filePathPattern.replace('[lang]', lang); - const fileExists = await fs.stat(filePath).then(() => true).catch(() => false); - if (!fileExists) { - return {}; - } else { - const fileContent = await fs.readFile(filePath, 'utf8'); - const langData = YAML.parse(fileContent) as LangDataNode; - const result = await this.flatten(langData, lang); - return result; - } - } - - async saveLangJson(filePathPattern: string, lang: string, record: Record): Promise { - await this.validatePath(filePathPattern); - - const langData = await this.unflatten(record, lang); - - const filePath = filePathPattern.replace('[lang]', lang); - const fileContent = YAML.stringify(langData); - // Create all directories in the path if they don't exist - const dirPath = path.dirname(filePath); - await fs.mkdir(dirPath, { recursive: true }); - // Write the file - await fs.writeFile(filePath, fileContent); - } -} From 61ecd1d0a7e8aa945d2756e6612128afac691323 Mon Sep 17 00:00:00 2001 From: maxprilutskiy Date: Thu, 2 May 2024 17:34:35 +0200 Subject: [PATCH 18/24] chore: cleanup --- packages/cli/src/auth.ts | 29 ++++++-------- packages/cli/src/i18n.ts | 6 +-- packages/cli/src/services/api-key.ts | 14 ------- packages/cli/src/services/auth.ts | 2 +- packages/cli/src/services/env.ts | 13 ------ packages/cli/src/services/settings.ts | 58 ++++++++++++++++++--------- 6 files changed, 54 insertions(+), 68 deletions(-) delete mode 100644 packages/cli/src/services/api-key.ts delete mode 100644 packages/cli/src/services/env.ts diff --git a/packages/cli/src/auth.ts b/packages/cli/src/auth.ts index 2c2563e1d..4e2619312 100644 --- a/packages/cli/src/auth.ts +++ b/packages/cli/src/auth.ts @@ -4,9 +4,7 @@ import express from 'express'; import cors from 'cors'; import open from 'open'; import readline from 'readline/promises'; -import { loadSettings } from "./services/settings.js"; -import { getEnv } from "./services/env.js"; -import { saveApiKey } from "./services/api-key.js"; +import { loadSettings, saveSettings } from "./services/settings.js"; import { loadAuth } from "./services/auth.js"; export default new Command() @@ -17,20 +15,22 @@ export default new Command() .option("--login", "Authenticate with Replexica API") .action(async (options) => { try { - const env = getEnv(); - let config = await loadSettings(); + let settings = await loadSettings(); if (options.logout) { - await logout(); + settings.auth.apiKey = null; + await saveSettings(settings); } if (options.login) { - await login(env.REPLEXICA_WEB_URL); - config = await loadSettings(); + const apiKey = await login(settings.auth.apiUrl); + settings.auth.apiKey = apiKey; + await saveSettings(settings); + settings = await loadSettings(); } const auth = await loadAuth({ - apiUrl: env.REPLEXICA_API_URL, - apiKey: config.auth.apiKey!, + apiUrl: settings.auth.apiUrl, + apiKey: settings.auth.apiKey!, }); if (!auth) { Ora().warn('Not authenticated'); @@ -43,12 +43,6 @@ export default new Command() } }); -async function logout() { - const spinner = Ora().start('Logging out'); - await saveApiKey(null); - spinner.succeed('Logged out'); -} - async function login(apiUrl: string) { await readline.createInterface({ input: process.stdin, @@ -60,7 +54,8 @@ async function login(apiUrl: string) { await open(`${apiUrl}/app/cli?port=${port}`, { wait: false }); }); spinner.succeed('API key received'); - await saveApiKey(apiKey); + + return apiKey; } async function waitForApiKey(cb: (port: string) => void): Promise { diff --git a/packages/cli/src/i18n.ts b/packages/cli/src/i18n.ts index e5df1f8ab..b7e8f69e0 100644 --- a/packages/cli/src/i18n.ts +++ b/packages/cli/src/i18n.ts @@ -1,6 +1,5 @@ import { Command } from 'commander'; import Ora from 'ora'; -import { getEnv } from './services/env.js'; import Z from 'zod'; import { loadConfig } from './services/config.js'; import { createBucketProcessor, createTranslator } from './services/bucket/core.js'; @@ -19,12 +18,11 @@ export default new Command() try { const flags = await loadFlags(options); const settings = await loadSettings(); - const env = getEnv(); const config = await loadConfiguration(); spinner = Ora().start('Authenticating...'); const auth = await loadAuth({ - apiUrl: env.REPLEXICA_API_URL, + apiUrl: settings.auth.apiUrl, apiKey: settings.auth.apiKey!, }); spinner.succeed(`Authenticated as ${auth.email}.`); @@ -48,7 +46,7 @@ export default new Command() for (const targetLocale of targetLocales) { bucketSpinner.start(`Translating from ${sourceLocale} to ${targetLocale}...`); const translatorFn = createTranslator({ - apiUrl: env.REPLEXICA_API_URL, + apiUrl: settings.auth.apiUrl, apiKey: settings.auth.apiKey!, skipCache: flags.skipCache, cacheOnly: flags.cacheOnly, diff --git a/packages/cli/src/services/api-key.ts b/packages/cli/src/services/api-key.ts deleted file mode 100644 index 28a531b66..000000000 --- a/packages/cli/src/services/api-key.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { getEnv } from "./env.js"; -import { loadSettings, saveSettings } from "./settings.js"; - -export async function saveApiKey(apiKey: string | null) { - const settings = await loadSettings(); - settings.auth.apiKey = apiKey; - await saveSettings(settings); -} - -export async function loadApiKey() { - const env = getEnv(); - const settings = await loadSettings(); - return env.REPLEXICA_API_KEY || settings.auth.apiKey; -} \ No newline at end of file diff --git a/packages/cli/src/services/auth.ts b/packages/cli/src/services/auth.ts index 57209064b..410e9aa24 100644 --- a/packages/cli/src/services/auth.ts +++ b/packages/cli/src/services/auth.ts @@ -6,7 +6,7 @@ export type LoadAuthParams = { export async function loadAuth(params: LoadAuthParams) { const whoami = await fetchWhoami(params.apiUrl, params.apiKey); if (!whoami) { - throw new Error("Failed to authenticate"); + throw new Error(`Not authenticated. Please login using 'replexica auth --login'.`); } return { diff --git a/packages/cli/src/services/env.ts b/packages/cli/src/services/env.ts deleted file mode 100644 index 6106ef7e6..000000000 --- a/packages/cli/src/services/env.ts +++ /dev/null @@ -1,13 +0,0 @@ -import Z from 'zod'; -import dotenv from 'dotenv'; - -const EnvSchema = Z.object({ - REPLEXICA_API_KEY: Z.string().optional(), - REPLEXICA_API_URL: Z.string().default('https://engine.replexica.com'), - REPLEXICA_WEB_URL: Z.string().default('https://replexica.com'), -}); - -export function getEnv() { - dotenv.config(); - return EnvSchema.parse(process.env); -} diff --git a/packages/cli/src/services/settings.ts b/packages/cli/src/services/settings.ts index ee1adbebb..2e4e4142c 100644 --- a/packages/cli/src/services/settings.ts +++ b/packages/cli/src/services/settings.ts @@ -3,39 +3,59 @@ import fs from 'fs'; import Ini from 'ini'; import os from 'os'; import path from 'path'; +import dotenv from 'dotenv'; +import _ from 'lodash'; const settingsFile = ".replexicarc"; const homedir = os.homedir(); const settingsFilePath = path.join(homedir, settingsFile); -const SettingsFileSchema = Z.object({ +const settingsSchema = Z.object({ auth: Z.object({ apiKey: Z.string().nullable(), + apiUrl: Z.string().default('https://engine.replexica.com'), + webUrl: Z.string().default('https://replexica.com'), }), }); -export async function loadSettings() { - const authFileExists = fs.existsSync(settingsFilePath); - let rawSettings = createEmptySettings(); - - if (authFileExists) { - const fileContents = fs.readFileSync(settingsFilePath, "utf8"); - rawSettings = Ini.parse(fileContents) as any; - } - const settings = SettingsFileSchema.parse(rawSettings); - - return settings; +function getEnv() { + dotenv.config(); + return Z.object({ + REPLEXICA_API_KEY: Z.string().optional(), + REPLEXICA_API_URL: Z.string().optional(), + REPLEXICA_WEB_URL: Z.string().optional(), + }) + .passthrough() + .parse(process.env); } -function createEmptySettings(): Z.infer { - return { +export async function loadSettings() { + const settingsFileData = await loadSettingsFile(); + const env = getEnv(); + + const result = settingsSchema.parse({ auth: { - apiKey: null, + apiKey: env.REPLEXICA_API_KEY || settingsFileData.auth?.apiKey, + apiUrl: env.REPLEXICA_API_URL || settingsFileData.auth?.apiUrl, + webUrl: env.REPLEXICA_WEB_URL || settingsFileData.auth?.webUrl, }, - }; + }); + return result; } -export async function saveSettings(config: Z.infer) { - const serialized = Ini.stringify(config); - fs.writeFileSync(settingsFilePath, serialized); +export async function saveSettings(settings: Z.infer) { + const settingsFileData = await loadSettingsFile(); + const newSettings = _.merge(settingsFileData, settings); + const content = Ini.stringify(newSettings); + + fs.writeFileSync(settingsFilePath, content); +} + +async function loadSettingsFile() { + try { + const content = fs.readFileSync(settingsFilePath, 'utf-8'); + return Ini.parse(content); + } catch (e) { + return {}; + } } From cc3094b4633a12fba2a76792b64fb998cdc856de Mon Sep 17 00:00:00 2001 From: maxprilutskiy Date: Thu, 2 May 2024 18:54:27 +0200 Subject: [PATCH 19/24] feat(spec): add add config / formats to @replexica/spec --- packages/spec/src/config.ts | 24 ++++++++++++++++++++++++ packages/spec/src/formats.ts | 4 ++-- packages/spec/src/index.ts | 1 + 3 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 packages/spec/src/config.ts diff --git a/packages/spec/src/config.ts b/packages/spec/src/config.ts new file mode 100644 index 000000000..90e34ecfe --- /dev/null +++ b/packages/spec/src/config.ts @@ -0,0 +1,24 @@ +import Z from 'zod'; +import { sourceLocaleSchema, targetLocaleSchema } from './locales'; +import { bucketTypeSchema } from './formats'; + +const localeSchema = Z.object({ + source: sourceLocaleSchema, + targets: Z.array(targetLocaleSchema), +}); + +export const configFileSchema = Z.object({ + version: Z.literal(1), + debug: Z.boolean().default(false).optional(), + locale: localeSchema, + buckets: Z.record(Z.string(), bucketTypeSchema).default({}).optional(), +}); + +export const defaultConfig: Z.infer = { + version: 1, + locale: { + source: 'en', + targets: ['es'], + }, + buckets: {}, +}; diff --git a/packages/spec/src/formats.ts b/packages/spec/src/formats.ts index 254d22689..012f69e11 100644 --- a/packages/spec/src/formats.ts +++ b/packages/spec/src/formats.ts @@ -1,5 +1,5 @@ import Z from 'zod'; -export const contentTypes = ['json', 'markdown', 'yaml', 'xcode', 'yaml-root-key'] as const; +export const bucketTypes = ['replexica', 'json', 'markdown', 'yaml', 'xcode', 'yaml-root-key'] as const; -export const contentTypeSchema = Z.enum(contentTypes); +export const bucketTypeSchema = Z.enum(bucketTypes); diff --git a/packages/spec/src/index.ts b/packages/spec/src/index.ts index 6c64b3d7d..43e09b618 100644 --- a/packages/spec/src/index.ts +++ b/packages/spec/src/index.ts @@ -1,2 +1,3 @@ export * from './locales'; export * from './formats'; +export * from './config'; \ No newline at end of file From 87deb92e6d774a1f61175db336ada0e2851b6035 Mon Sep 17 00:00:00 2001 From: maxprilutskiy Date: Thu, 2 May 2024 18:54:57 +0200 Subject: [PATCH 20/24] feat(cli): add `replexica config` cli command --- packages/cli/src/config.ts | 30 ++++++++++++++++++++++++++++++ packages/cli/src/index.ts | 2 ++ 2 files changed, 32 insertions(+) create mode 100644 packages/cli/src/config.ts diff --git a/packages/cli/src/config.ts b/packages/cli/src/config.ts new file mode 100644 index 000000000..77ddc260d --- /dev/null +++ b/packages/cli/src/config.ts @@ -0,0 +1,30 @@ +import { Command } from "commander"; +import _ from "lodash"; +import fs from 'fs'; +import path from 'path'; +import { defaultConfig } from "@replexica/spec"; + +export default new Command() + .command("config") + .description("Prints out the current configuration") + .helpOption("-h, --help", "Show help") + .action(async (options) => { + const fileConfig = loadReplexicaFileConfig(); + const config = _.merge( + {}, + defaultConfig, + fileConfig, + ); + + console.log(JSON.stringify(config, null, 2)); + }); + +function loadReplexicaFileConfig(): any { + const replexicaConfigPath = path.resolve(process.cwd(), 'i18n.json'); + const fileExists = fs.existsSync(replexicaConfigPath); + if (!fileExists) { return undefined; } + + const fileContent = fs.readFileSync(replexicaConfigPath, 'utf-8'); + const replexicaFileConfig = JSON.parse(fileContent); + return replexicaFileConfig; +} \ No newline at end of file diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 98613c7ed..8e5755444 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -3,6 +3,7 @@ import { Command } from 'commander'; import i18nCmd from './i18n.js'; import authCmd from './auth.js'; import initCmd from './init.js'; +import configCmd from './config.js'; export default new Command() .name('replexica') @@ -11,4 +12,5 @@ export default new Command() .addCommand(i18nCmd) .addCommand(authCmd) .addCommand(initCmd) + .addCommand(configCmd) .parse(process.argv); From b4e179700e2f3220a4e927045eeea39e0c93a43b Mon Sep 17 00:00:00 2001 From: maxprilutskiy Date: Thu, 2 May 2024 18:55:28 +0200 Subject: [PATCH 21/24] feat(cli): fallback to using default config instead of throwing an error --- packages/cli/src/i18n.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/cli/src/i18n.ts b/packages/cli/src/i18n.ts index b7e8f69e0..4c2ca0a0a 100644 --- a/packages/cli/src/i18n.ts +++ b/packages/cli/src/i18n.ts @@ -1,10 +1,11 @@ import { Command } from 'commander'; import Ora from 'ora'; import Z from 'zod'; -import { loadConfig } from './services/config.js'; +import { loadConfig, saveConfig } from './services/config.js'; import { createBucketProcessor, createTranslator } from './services/bucket/core.js'; import { loadSettings } from './services/settings.js'; import { loadAuth } from './services/auth.js'; +import { defaultConfig } from '@replexica/spec'; export default new Command() .command('i18n') @@ -85,9 +86,13 @@ async function loadFlags(options: any) { } async function loadConfiguration() { - const config = await loadConfig(); + const spinner = Ora().start('Loading i18n configuration...'); + let config = await loadConfig(); if (!config) { - throw new Error(`Couldn't load i18n configuration. Please run 'replexica init' to initialize your Replexica project.`); + config = defaultConfig; + spinner.succeed('No i18n.json config found. Using default configuration.'); + } else { + spinner.succeed('Configuration loaded.'); } return config; } From d6d9bd464a09c101f5f4c144334833508c0c266a Mon Sep 17 00:00:00 2001 From: maxprilutskiy Date: Thu, 2 May 2024 18:56:05 +0200 Subject: [PATCH 22/24] feat(compiler): use config merging in compiler --- packages/compiler/src/plugins/next.ts | 70 +++++++++++++++++---------- 1 file changed, 45 insertions(+), 25 deletions(-) diff --git a/packages/compiler/src/plugins/next.ts b/packages/compiler/src/plugins/next.ts index 1c0b6649c..ef2885ab7 100644 --- a/packages/compiler/src/plugins/next.ts +++ b/packages/compiler/src/plugins/next.ts @@ -1,33 +1,53 @@ import walk from 'ignore-walk'; import fs from 'fs'; +import _ from 'lodash'; import path from 'path'; import { basePlugin, shouldTransformFile, transformFile } from './base'; -import { ReplexicaConfig } from '../options'; +import { configFileSchema, defaultConfig } from '@replexica/spec'; -export function nextPlugin(replexicaOptions: O, nextConfig: any) { - return { - ...nextConfig, - webpack(config: any, ctx: any) { - const plugin = basePlugin.webpack(replexicaOptions); - config.plugins.unshift(plugin); - - const files = walk.sync({ - path: ctx.dir, - ignoreFiles: ['.gitignore'], - }) - .map((relFilePath) => path.resolve(ctx.dir, relFilePath)) - .filter((file) => shouldTransformFile(file)); - - for (const file of files) { - const code = fs.readFileSync(file, 'utf-8'); - transformFile(code, file, replexicaOptions); - } +export function nextPlugin(partialReplexicaConfig?: Partial) { + const replexicaFileConfig = loadReplexicaFileConfig(); + const finalReplexicaConfig = _.merge( + {}, + defaultConfig, + replexicaFileConfig, + partialReplexicaConfig, + ); + return (nextConfig: any) => { + return { + ...nextConfig, + webpack(config: any, ctx: any) { + const plugin = basePlugin.webpack(finalReplexicaConfig); + config.plugins.unshift(plugin); + + const files = walk.sync({ + path: ctx.dir, + ignoreFiles: ['.gitignore'], + }) + .map((relFilePath) => path.resolve(ctx.dir, relFilePath)) + .filter((file) => shouldTransformFile(file)); + + for (const file of files) { + const code = fs.readFileSync(file, 'utf-8'); + transformFile(code, file, finalReplexicaConfig); + } + + if (typeof nextConfig.webpack === 'function') { + return nextConfig.webpack(config, ctx); + } + + return config; + }, + }; + } +} - if (typeof nextConfig.webpack === 'function') { - return nextConfig.webpack(config, ctx); - } +function loadReplexicaFileConfig(): any { + const replexicaConfigPath = path.resolve(process.cwd(), 'i18n.json'); + const fileExists = fs.existsSync(replexicaConfigPath); + if (!fileExists) { return undefined; } - return config; - }, - }; + const fileContent = fs.readFileSync(replexicaConfigPath, 'utf-8'); + const replexicaFileConfig = JSON.parse(fileContent); + return replexicaFileConfig; } From 138a9e1ae6341ece713ab5c581d44db490335740 Mon Sep 17 00:00:00 2001 From: maxprilutskiy Date: Thu, 2 May 2024 18:56:23 +0200 Subject: [PATCH 23/24] feat: migrate to the new compiler / config --- demo/next-app-shadcnui/next.config.mjs | 16 ++------------ demo/next-app/next.config.mjs | 16 ++------------ packages/cli/src/init.ts | 6 +++--- packages/cli/src/services/bucket/core.ts | 8 +------ packages/cli/src/services/config.ts | 27 ++---------------------- packages/compiler/package.json | 4 +++- pnpm-lock.yaml | 6 ++++++ 7 files changed, 19 insertions(+), 64 deletions(-) diff --git a/demo/next-app-shadcnui/next.config.mjs b/demo/next-app-shadcnui/next.config.mjs index 79275663b..0d4b84a0a 100644 --- a/demo/next-app-shadcnui/next.config.mjs +++ b/demo/next-app-shadcnui/next.config.mjs @@ -1,18 +1,6 @@ -import compiler from '@replexica/compiler'; +import replexica from '@replexica/compiler'; /** @type {import('next').NextConfig} */ const nextConfig = {}; -/** @type {import('@replexica/compiler').ReplexicaConfig} */ -const replexicaConfig = { - locale: { - source: 'en', - targets: ['es'], - }, - debug: true, -}; - -export default compiler.next( - replexicaConfig, - nextConfig, -); +export default replexica.next()(nextConfig); diff --git a/demo/next-app/next.config.mjs b/demo/next-app/next.config.mjs index 79275663b..0d4b84a0a 100644 --- a/demo/next-app/next.config.mjs +++ b/demo/next-app/next.config.mjs @@ -1,18 +1,6 @@ -import compiler from '@replexica/compiler'; +import replexica from '@replexica/compiler'; /** @type {import('next').NextConfig} */ const nextConfig = {}; -/** @type {import('@replexica/compiler').ReplexicaConfig} */ -const replexicaConfig = { - locale: { - source: 'en', - targets: ['es'], - }, - debug: true, -}; - -export default compiler.next( - replexicaConfig, - nextConfig, -); +export default replexica.next()(nextConfig); diff --git a/packages/cli/src/init.ts b/packages/cli/src/init.ts index ed0cb93f0..b66632aad 100644 --- a/packages/cli/src/init.ts +++ b/packages/cli/src/init.ts @@ -1,6 +1,7 @@ import { Command } from "commander"; import Ora from 'ora'; -import { createEmptyConfig, loadConfig, saveConfig } from "./services/config.js"; +import { loadConfig, saveConfig } from "./services/config.js"; +import { defaultConfig } from "@replexica/spec"; export default new Command() .command("init") @@ -15,8 +16,7 @@ export default new Command() return process.exit(1); } - config = await createEmptyConfig(); - await saveConfig(config); + await saveConfig(defaultConfig); spinner.succeed('Replexica project initialized'); }); diff --git a/packages/cli/src/services/bucket/core.ts b/packages/cli/src/services/bucket/core.ts index 1bb802459..bbf094958 100644 --- a/packages/cli/src/services/bucket/core.ts +++ b/packages/cli/src/services/bucket/core.ts @@ -1,7 +1,6 @@ import _ from 'lodash'; -import Z from 'zod'; import { ReplexicaBucketProcessor } from './replexica.js'; -import { contentTypes, contentTypeSchema } from '@replexica/spec'; +import { bucketTypeSchema } from '@replexica/spec'; import { JsonBucketProcessor } from './json.js'; import { createId } from '@paralleldrive/cuid2'; import { YamlBucketProcessor } from './yaml.js'; @@ -25,11 +24,6 @@ export interface IBucketProcessor { save(locale: string, payload: BucketPayload): Promise; } - -export const bucketTypes = [...contentTypes, 'replexica'] as const; - -export const bucketTypeSchema = Z.union([Z.literal('replexica'), contentTypeSchema]); - export function createBucketProcessor( bucketType: typeof bucketTypeSchema._type, bucketPath: string, diff --git a/packages/cli/src/services/config.ts b/packages/cli/src/services/config.ts index 998cf562b..912ba5d11 100644 --- a/packages/cli/src/services/config.ts +++ b/packages/cli/src/services/config.ts @@ -1,24 +1,11 @@ import Z from 'zod'; import fs from 'fs'; import path from 'path'; -import { sourceLocaleSchema, targetLocaleSchema } from '@replexica/spec'; -import { bucketTypeSchema } from './bucket/core.js'; +import { configFileSchema } from '@replexica/spec'; const configFile = "i18n.json"; const configFilePath = path.join(process.cwd(), configFile); -const localeSchema = Z.object({ - source: sourceLocaleSchema, - targets: Z.array(targetLocaleSchema), -}); - -const configFileSchema = Z.object({ - version: Z.literal(1), - debug: Z.boolean().default(false).optional(), - locale: localeSchema, - buckets: Z.record(Z.string(), bucketTypeSchema).default({}).optional(), -}); - export async function loadConfig(): Promise | null> { const configFileExists = await fs.existsSync(configFilePath); if (!configFileExists) { return null; } @@ -30,18 +17,8 @@ export async function loadConfig(): Promise | n return config; } -export async function createEmptyConfig(): Promise> { - return { - version: 1, - locale: { - source: 'en', - targets: ['es'], - }, - buckets: {}, - }; -} - export async function saveConfig(config: Z.infer) { const serialized = JSON.stringify(config, null, 2); fs.writeFileSync(configFilePath, serialized); + return config; } diff --git a/packages/compiler/package.json b/packages/compiler/package.json index 0738281de..fcba43bc2 100644 --- a/packages/compiler/package.json +++ b/packages/compiler/package.json @@ -19,12 +19,13 @@ "author": "", "license": "ISC", "dependencies": { - "@replexica/spec": "workspace:*", "@babel/core": "^7.24.4", "@babel/generator": "^7.24.4", "@babel/parser": "^7.24.4", "@babel/types": "^7.24.0", + "@replexica/spec": "workspace:*", "ignore-walk": "^6.0.4", + "lodash": "^4.17.21", "object-hash": "^3.0.0", "typescript": "^5.4.5", "unplugin": "^1.10.1", @@ -35,6 +36,7 @@ "@types/babel__core": "^7.20.5", "@types/babel__generator": "^7.6.8", "@types/ignore-walk": "^4.0.3", + "@types/lodash": "^4.17.0", "@types/object-hash": "^3.0.6", "unbuild": "^2.0.0" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 53b102aff..9d1253bf9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -271,6 +271,9 @@ importers: ignore-walk: specifier: ^6.0.4 version: 6.0.4 + lodash: + specifier: ^4.17.21 + version: 4.17.21 object-hash: specifier: ^3.0.0 version: 3.0.0 @@ -296,6 +299,9 @@ importers: '@types/ignore-walk': specifier: ^4.0.3 version: 4.0.3 + '@types/lodash': + specifier: ^4.17.0 + version: 4.17.0 '@types/object-hash': specifier: ^3.0.6 version: 3.0.6 From fb95d4062193f2f6c309b596c29bf4c59e00e092 Mon Sep 17 00:00:00 2001 From: maxprilutskiy Date: Thu, 2 May 2024 19:00:13 +0200 Subject: [PATCH 24/24] docs(spec): changeset for `@replexica/spec` --- .changeset/happy-squids-watch.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/happy-squids-watch.md diff --git a/.changeset/happy-squids-watch.md b/.changeset/happy-squids-watch.md new file mode 100644 index 000000000..db27f0e2e --- /dev/null +++ b/.changeset/happy-squids-watch.md @@ -0,0 +1,5 @@ +--- +"@replexica/spec": minor +--- + +intro a `@replexica/spec` package containing common definitions, constants, schemas, and types