Skip to content

Commit

Permalink
feat(ivy): i18n - support inlining of XTB formatted translation files (
Browse files Browse the repository at this point in the history
…#33444)

This commit implements the `XtbTranslationParser`, which can read XTB
formatted files.

PR Close #33444
  • Loading branch information
petebacondarwin authored and AndrewKushnir committed Oct 28, 2019
1 parent c2f13a1 commit 2c623fd
Show file tree
Hide file tree
Showing 6 changed files with 424 additions and 4 deletions.
2 changes: 2 additions & 0 deletions packages/localize/src/tools/src/translate/main.ts
Expand Up @@ -18,6 +18,7 @@ import {TranslationLoader} from './translation_files/translation_loader';
import {SimpleJsonTranslationParser} from './translation_files/translation_parsers/simple_json_translation_parser';
import {Xliff1TranslationParser} from './translation_files/translation_parsers/xliff1_translation_parser';
import {Xliff2TranslationParser} from './translation_files/translation_parsers/xliff2_translation_parser';
import {XtbTranslationParser} from './translation_files/translation_parsers/xtb_translation_parser';
import {Translator} from './translator';
import {Diagnostics} from '../diagnostics';

Expand Down Expand Up @@ -141,6 +142,7 @@ export function translateFiles({sourceRootPath, sourceFilePaths, translationFile
[
new Xliff2TranslationParser(),
new Xliff1TranslationParser(),
new XtbTranslationParser(diagnostics),
new SimpleJsonTranslationParser(),
],
diagnostics);
Expand Down
Expand Up @@ -27,6 +27,9 @@ export function parseInnerRange(element: Element): Node[] {
const xml = xmlParser.parse(
element.sourceSpan.start.file.content, element.sourceSpan.start.file.url,
{tokenizeExpansionForms: true, range: getInnerRange(element)});
if (xml.errors.length) {
throw xml.errors.map(e => new TranslationParseError(e.span, e.msg).toString()).join('\n');
}
return xml.rootNodes;
}

Expand Down
@@ -0,0 +1,103 @@
/**
* @license
* Copyright Google Inc. All Rights Reserved.
*
* Use of this source code is governed by an MIT-style license that can be
* found in the LICENSE file at https://angular.io/license
*/
import {Element, Node, XmlParser, visitAll} from '@angular/compiler';
import {ɵParsedTranslation} from '@angular/localize';
import {extname} from 'path';

import {Diagnostics} from '../../../diagnostics';
import {BaseVisitor} from '../base_visitor';
import {MessageSerializer} from '../message_serialization/message_serializer';
import {TargetMessageRenderer} from '../message_serialization/target_message_renderer';

import {TranslationParseError} from './translation_parse_error';
import {ParsedTranslationBundle, TranslationParser} from './translation_parser';
import {getAttrOrThrow, parseInnerRange} from './translation_utils';



/**
* A translation parser that can load XB files.
*/
export class XtbTranslationParser implements TranslationParser {
constructor(private diagnostics: Diagnostics) {}

canParse(filePath: string, contents: string): boolean {
const extension = extname(filePath);
return (extension === '.xtb' || extension === '.xmb') &&
contents.includes('<translationbundle');
}

parse(filePath: string, contents: string): ParsedTranslationBundle {
const xmlParser = new XmlParser();
const xml = xmlParser.parse(contents, filePath);
const bundle = XtbVisitor.extractBundle(this.diagnostics, xml.rootNodes);
if (bundle === undefined) {
throw new Error(`Unable to parse "${filePath}" as XTB/XMB format.`);
}
return bundle;
}
}

class XtbVisitor extends BaseVisitor {
static extractBundle(diagnostics: Diagnostics, messageBundles: Node[]): ParsedTranslationBundle
|undefined {
const visitor = new this(diagnostics);
const bundles: ParsedTranslationBundle[] = visitAll(visitor, messageBundles, undefined);
return bundles[0];
}

constructor(private diagnostics: Diagnostics) { super(); }

visitElement(element: Element, bundle: ParsedTranslationBundle|undefined): any {
switch (element.name) {
case 'translationbundle':
if (bundle) {
throw new TranslationParseError(
element.sourceSpan, '<translationbundle> elements can not be nested');
}
const langAttr = element.attrs.find((attr) => attr.name === 'lang');
bundle = {locale: langAttr && langAttr.value, translations: {}};
visitAll(this, element.children, bundle);
return bundle;

case 'translation':
if (!bundle) {
throw new TranslationParseError(
element.sourceSpan, '<translation> must be inside a <translationbundle>');
}
const id = getAttrOrThrow(element, 'id');
if (bundle.translations.hasOwnProperty(id)) {
throw new TranslationParseError(
element.sourceSpan, `Duplicated translations for message "${id}"`);
} else {
try {
bundle.translations[id] = serializeTargetMessage(element);
} catch (error) {
if (typeof error === 'string') {
this.diagnostics.warn(
`Could not parse message with id "${id}" - perhaps it has an unrecognised ICU format?\n` +
error);
} else {
throw error;
}
}
}
break;

default:
throw new TranslationParseError(element.sourceSpan, 'Unexpected tag');
}
}
}

function serializeTargetMessage(source: Element): ɵParsedTranslation {
const serializer = new MessageSerializer(
new TargetMessageRenderer(),
{inlineElements: [], placeholder: {elementName: 'ph', nameAttribute: 'name'}});
return serializer.serialize(parseInnerRange(source));
}
@@ -0,0 +1,13 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE translationbundle [<!ELEMENT translationbundle (translation)*>
<!ATTLIST translationbundle lang CDATA #REQUIRED>

<!ELEMENT translation (#PCDATA|ph)*>
<!ATTLIST translation id CDATA #REQUIRED>

<!ELEMENT ph EMPTY>
<!ATTLIST ph name CDATA #REQUIRED>
]>
<translationbundle lang="it">
<translation id="3291030485717846467">Ciao, <ph name="PH"/>!</translation>
</translationbundle>
Expand Up @@ -29,7 +29,8 @@ describe('translateFiles()', () => {
sourceFilePaths: resolveAll(__dirname + '/test_files', ['test-1.txt', 'test-2.txt']),
outputPathFn,
translationFilePaths: resolveAll(
__dirname + '/locales', ['messages.de.json', 'messages.es.xlf', 'messages.fr.xlf']),
__dirname + '/locales',
['messages.de.json', 'messages.es.xlf', 'messages.fr.xlf', 'messages.it.xtb']),
translationFileLocales: [], diagnostics,
missingTranslation: 'error'
});
Expand All @@ -48,6 +49,10 @@ describe('translateFiles()', () => {
.toEqual('Contents of test-1.txt');
expect(FileUtils.readFile(resolve(testDir, 'es', 'test-2.txt')))
.toEqual('Contents of test-2.txt');
expect(FileUtils.readFile(resolve(testDir, 'it', 'test-1.txt')))
.toEqual('Contents of test-1.txt');
expect(FileUtils.readFile(resolve(testDir, 'it', 'test-2.txt')))
.toEqual('Contents of test-2.txt');
});

it('should translate and copy source-code files to the destination folders', () => {
Expand All @@ -57,7 +62,8 @@ describe('translateFiles()', () => {
sourceRootPath: resolve(__dirname, 'test_files'),
sourceFilePaths: resolveAll(__dirname + '/test_files', ['test.js']), outputPathFn,
translationFilePaths: resolveAll(
__dirname + '/locales', ['messages.de.json', 'messages.es.xlf', 'messages.fr.xlf']),
__dirname + '/locales',
['messages.de.json', 'messages.es.xlf', 'messages.fr.xlf', 'messages.it.xtb']),
translationFileLocales: [], diagnostics,
missingTranslation: 'error',
});
Expand All @@ -70,6 +76,8 @@ describe('translateFiles()', () => {
.toEqual(`var name="World";var message="Guten Tag, "+name+"!";`);
expect(FileUtils.readFile(resolve(testDir, 'es', 'test.js')))
.toEqual(`var name="World";var message="Hola, "+name+"!";`);
expect(FileUtils.readFile(resolve(testDir, 'it', 'test.js')))
.toEqual(`var name="World";var message="Ciao, "+name+"!";`);
});

it('should translate and copy source-code files overriding the locales', () => {
Expand All @@ -79,7 +87,8 @@ describe('translateFiles()', () => {
sourceRootPath: resolve(__dirname, 'test_files'),
sourceFilePaths: resolveAll(__dirname + '/test_files', ['test.js']), outputPathFn,
translationFilePaths: resolveAll(
__dirname + '/locales', ['messages.de.json', 'messages.es.xlf', 'messages.fr.xlf']),
__dirname + '/locales',
['messages.de.json', 'messages.es.xlf', 'messages.fr.xlf', 'messages.it.xtb']),
translationFileLocales: ['xde', undefined, 'fr'], diagnostics,
missingTranslation: 'error',
});
Expand All @@ -97,6 +106,8 @@ describe('translateFiles()', () => {
.toEqual(`var name="World";var message="Hola, "+name+"!";`);
expect(FileUtils.readFile(resolve(testDir, 'fr', 'test.js')))
.toEqual(`var name="World";var message="Bonjour, "+name+"!";`);
expect(FileUtils.readFile(resolve(testDir, 'it', 'test.js')))
.toEqual(`var name="World";var message="Ciao, "+name+"!";`);
});

it('should transform and/or copy files to the destination folders', () => {
Expand All @@ -108,7 +119,8 @@ describe('translateFiles()', () => {
resolveAll(__dirname + '/test_files', ['test-1.txt', 'test-2.txt', 'test.js']),
outputPathFn,
translationFilePaths: resolveAll(
__dirname + '/locales', ['messages.de.json', 'messages.es.xlf', 'messages.fr.xlf']),
__dirname + '/locales',
['messages.de.json', 'messages.es.xlf', 'messages.fr.xlf', 'messages.it.xtb']),
translationFileLocales: [], diagnostics,
missingTranslation: 'error',
});
Expand All @@ -127,13 +139,19 @@ describe('translateFiles()', () => {
.toEqual('Contents of test-1.txt');
expect(FileUtils.readFile(resolve(testDir, 'es', 'test-2.txt')))
.toEqual('Contents of test-2.txt');
expect(FileUtils.readFile(resolve(testDir, 'it', 'test-1.txt')))
.toEqual('Contents of test-1.txt');
expect(FileUtils.readFile(resolve(testDir, 'it', 'test-2.txt')))
.toEqual('Contents of test-2.txt');

expect(FileUtils.readFile(resolve(testDir, 'fr', 'test.js')))
.toEqual(`var name="World";var message="Bonjour, "+name+"!";`);
expect(FileUtils.readFile(resolve(testDir, 'de', 'test.js')))
.toEqual(`var name="World";var message="Guten Tag, "+name+"!";`);
expect(FileUtils.readFile(resolve(testDir, 'es', 'test.js')))
.toEqual(`var name="World";var message="Hola, "+name+"!";`);
expect(FileUtils.readFile(resolve(testDir, 'it', 'test.js')))
.toEqual(`var name="World";var message="Ciao, "+name+"!";`);
});
});

Expand Down

0 comments on commit 2c623fd

Please sign in to comment.