Skip to content

Commit

Permalink
feat(@formatjs/cli): add support for pseudo-locales, fix #2165
Browse files Browse the repository at this point in the history
  • Loading branch information
longlho committed Oct 1, 2020
1 parent 8ae60b8 commit 015bbdb
Show file tree
Hide file tree
Showing 6 changed files with 380 additions and 9 deletions.
5 changes: 5 additions & 0 deletions packages/cli/src/cli.ts
Expand Up @@ -163,6 +163,11 @@ If this is not provided, result will be printed to stdout`
'--ast',
`Whether to compile to AST. See https://formatjs.io/docs/guides/advanced-usage#pre-parsing-messages
for more information`
)
.option(
'--pseudo-locale <pseudoLocale>',
`Whether to generate pseudo-locale files. See http://localhost:3000/docs/tooling/cli#--pseudo-locale-pseudolocale for possible values.
"--ast" is required for this to work.`
)
.action(async (filePattern: string, opts: CompileCLIOpts) => {
const files = globSync(filePattern);
Expand Down
32 changes: 30 additions & 2 deletions packages/cli/src/compile.ts
Expand Up @@ -2,9 +2,17 @@ import {parse, MessageFormatElement} from 'intl-messageformat-parser';
import {outputFile, readJSON} from 'fs-extra';
import stringify from 'json-stable-stringify';
import {resolveBuiltinFormatter, Formatter} from './formatters';
import {
generateXXAC,
generateXXLS,
generateXXHA,
generateENXA,
} from './pseudo_locale';

export type CompileFn = (msgs: any) => Record<string, string>;

export type PseudoLocale = 'xx-LS' | 'xx-AC' | 'xx-HA' | 'en-XA';

export interface CompileCLIOpts extends Opts {
/**
* The target file that contains compiled messages.
Expand All @@ -21,6 +29,10 @@ export interface Opts {
* `Record<string, string>` so we can compile.
*/
format?: string | Formatter;
/**
* Whether to compile to pseudo locale
*/
pseudoLocale?: PseudoLocale;
}

/**
Expand All @@ -33,7 +45,7 @@ export interface Opts {
* @returns serialized result in string format
*/
export async function compile(inputFiles: string[], opts: Opts = {}) {
const {ast, format} = opts;
const {ast, format, pseudoLocale} = opts;
const formatter = await resolveBuiltinFormatter(format);

const messages: Record<string, string> = {};
Expand All @@ -56,7 +68,23 @@ Message from ${compiled[id]}: ${inputFile}
messages[id] = compiled[id];
try {
const msgAst = parse(compiled[id]);
messageAsts[id] = msgAst;
switch (pseudoLocale) {
case 'xx-LS':
messageAsts[id] = generateXXLS(msgAst);
break;
case 'xx-AC':
messageAsts[id] = generateXXAC(msgAst);
break;
case 'xx-HA':
messageAsts[id] = generateXXHA(msgAst);
break;
case 'en-XA':
messageAsts[id] = generateENXA(msgAst);
break;
default:
messageAsts[id] = msgAst;
break;
}
} catch (e) {
console.error(
`Error validating message "${compiled[id]}" with ID "${id}" in file "${inputFile}"`
Expand Down
76 changes: 76 additions & 0 deletions packages/cli/src/pseudo_locale.ts
@@ -0,0 +1,76 @@
import {
parse,
MessageFormatElement,
TYPE,
isLiteralElement,
isPluralElement,
isSelectElement,
} from 'intl-messageformat-parser';

export function generateXXLS(
msg: string | MessageFormatElement[]
): MessageFormatElement[] {
const ast = typeof msg === 'string' ? parse(msg) : msg;
const lastChunk = ast.pop();
if (lastChunk && isLiteralElement(lastChunk)) {
lastChunk.value += 'SSSSSSSSSSSSSSSSSSSSSSSSS';
return [...ast, lastChunk];
}
return [...ast, {type: TYPE.literal, value: 'SSSSSSSSSSSSSSSSSSSSSSSSS'}];
}

export function generateXXAC(
msg: string | MessageFormatElement[]
): MessageFormatElement[] {
const ast = typeof msg === 'string' ? parse(msg) : msg;
ast.forEach(el => {
if (isLiteralElement(el)) {
el.value = el.value.toUpperCase();
} else if (isPluralElement(el) || isSelectElement(el)) {
for (const opt of Object.values(el.options)) {
generateXXAC(opt.value);
}
}
});
return ast;
}

export function generateXXHA(
msg: string | MessageFormatElement[]
): MessageFormatElement[] {
const ast = typeof msg === 'string' ? parse(msg) : msg;
const firstChunk = ast.shift();
if (firstChunk && isLiteralElement(firstChunk)) {
firstChunk.value = '[javascript]' + firstChunk.value;
return [firstChunk, ...ast];
}
return [{type: TYPE.literal, value: '[javascript]'}, ...ast];
}

const ASCII = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
const ACCENTED_ASCII = 'âḃćḋèḟĝḫíĵǩĺṁńŏṗɋŕśṭůṿẘẋẏẓḀḂḈḊḔḞḠḢḬĴḴĻḾŊÕṔɊŔṠṮŨṼẄẌŸƵ';

export function generateENXA(
msg: string | MessageFormatElement[]
): MessageFormatElement[] {
const ast = typeof msg === 'string' ? parse(msg) : msg;
ast.forEach(el => {
if (isLiteralElement(el)) {
el.value = el.value
.split('')
.map(c => {
const i = ASCII.indexOf(c);
if (i < 0) {
return c;
}
return ACCENTED_ASCII[i];
})
.join('');
} else if (isPluralElement(el) || isSelectElement(el)) {
for (const opt of Object.values(el.options)) {
generateENXA(opt.value);
}
}
});
return ast;
}
219 changes: 212 additions & 7 deletions packages/cli/tests/compile/__snapshots__/integration.test.ts.snap
Expand Up @@ -74,7 +74,7 @@ We also verify that the messages are valid ICU and not malformed.
<translation_files> can be a glob like \\"foo/**/en.json\\"
Options:
--format <path> Path to a formatter file that converts \`<translation_file>\` to \`Record<string, string>\`
--format <path> Path to a formatter file that converts \`<translation_file>\` to \`Record<string, string>\`
so we can compile. The file must export a function named \`compile\` with the signature:
\`\`\`
type CompileFn = <T = Record<string, MessageDescriptor>>(
Expand All @@ -83,12 +83,17 @@ Options:
\`\`\`
This is especially useful to convert from a TMS-specific format back to react-intl format
--out-file <path> Compiled translation output file.
If this is not provided, result will be printed to stdout
--ast Whether to compile to AST. See
https://formatjs.io/docs/guides/advanced-usage#pre-parsing-messages
for more information
-h, --help display help for command
--out-file <path> Compiled translation output file.
If this is not provided, result will be
printed to stdout
--ast Whether to compile to AST. See
https://formatjs.io/docs/guides/advanced-usage#pre-parsing-messages
for more information
--pseudo-locale <pseudoLocale> Whether to generate pseudo-locale files. See
http://localhost:3000/docs/tooling/cli#--pseudo-locale-pseudolocale
for possible values.
\\"--ast\\" is required for this to work.
-h, --help display help for command
",
}
`;
Expand All @@ -108,6 +113,61 @@ Object {
}
`;
exports[`en-XA json 1`] = `
Object {
"stderr": "",
"stdout": "{
\\"a1d12\\": [
{
\\"type\\": 0,
\\"value\\": \\"Ḭ ḫâṿè \\"
},
{
\\"offset\\": 0,
\\"options\\": {
\\"one\\": {
\\"value\\": [
{
\\"type\\": 0,
\\"value\\": \\"â ḋŏĝ\\"
}
]
},
\\"other\\": {
\\"value\\": [
{
\\"type\\": 0,
\\"value\\": \\"ṁâńẏ ḋŏĝś\\"
}
]
}
},
\\"pluralType\\": \\"cardinal\\",
\\"type\\": 6,
\\"value\\": \\"count\\"
}
],
\\"a1dd2\\": [
{
\\"type\\": 0,
\\"value\\": \\"ṁẏ ńâṁè íś \\"
},
{
\\"type\\": 1,
\\"value\\": \\"name\\"
}
],
\\"ashd2\\": [
{
\\"type\\": 0,
\\"value\\": \\"â ṁèśśâĝè\\"
}
]
}
",
}
`;
exports[`normal json 1`] = `
Object {
"stderr": "",
Expand Down Expand Up @@ -264,3 +324,148 @@ Object {
"ashd2": "a message",
}
`;
exports[`xx-AC json 1`] = `
Object {
"stderr": "",
"stdout": "{
\\"a1d12\\": [
{
\\"type\\": 0,
\\"value\\": \\"I HAVE \\"
},
{
\\"offset\\": 0,
\\"options\\": {
\\"one\\": {
\\"value\\": [
{
\\"type\\": 0,
\\"value\\": \\"A DOG\\"
}
]
},
\\"other\\": {
\\"value\\": [
{
\\"type\\": 0,
\\"value\\": \\"MANY DOGS\\"
}
]
}
},
\\"pluralType\\": \\"cardinal\\",
\\"type\\": 6,
\\"value\\": \\"count\\"
}
],
\\"a1dd2\\": [
{
\\"type\\": 0,
\\"value\\": \\"MY NAME IS \\"
},
{
\\"type\\": 1,
\\"value\\": \\"name\\"
}
],
\\"ashd2\\": [
{
\\"type\\": 0,
\\"value\\": \\"A MESSAGE\\"
}
]
}
",
}
`;
exports[`xx-HA json 1`] = `
Object {
"stderr": "",
"stdout": "{
\\"a1d12\\": [
{
\\"type\\": 0,
\\"value\\": \\"[javascript]I have \\"
},
{
\\"offset\\": 0,
\\"options\\": {
\\"one\\": {
\\"value\\": [
{
\\"type\\": 0,
\\"value\\": \\"a dog\\"
}
]
},
\\"other\\": {
\\"value\\": [
{
\\"type\\": 0,
\\"value\\": \\"many dogs\\"
}
]
}
},
\\"pluralType\\": \\"cardinal\\",
\\"type\\": 6,
\\"value\\": \\"count\\"
}
],
\\"a1dd2\\": [
{
\\"type\\": 0,
\\"value\\": \\"[javascript]my name is \\"
},
{
\\"type\\": 1,
\\"value\\": \\"name\\"
}
],
\\"ashd2\\": [
{
\\"type\\": 0,
\\"value\\": \\"[javascript]a message\\"
}
]
}
",
}
`;
exports[`xx-LS json 1`] = `
Object {
"stderr": "",
"stdout": "{
\\"a1d12\\": [
{
\\"type\\": 0,
\\"value\\": \\"I have \\"
},
{
\\"type\\": 0,
\\"value\\": \\"SSSSSSSSSSSSSSSSSSSSSSSSS\\"
}
],
\\"a1dd2\\": [
{
\\"type\\": 0,
\\"value\\": \\"my name is \\"
},
{
\\"type\\": 0,
\\"value\\": \\"SSSSSSSSSSSSSSSSSSSSSSSSS\\"
}
],
\\"ashd2\\": [
{
\\"type\\": 0,
\\"value\\": \\"a messageSSSSSSSSSSSSSSSSSSSSSSSSS\\"
}
]
}
",
}
`;

0 comments on commit 015bbdb

Please sign in to comment.