Skip to content
Permalink
Browse files
Implement cross-channel localization support, add Focus for iOS and A…
…ndroid (#895)
  • Loading branch information
flodolo committed Oct 11, 2017
1 parent 28f2954 commit d1dbaa349333adb7f1d23195099d3b4dfeadc7e4
Show file tree
Hide file tree
Showing 69 changed files with 1,246 additions and 989 deletions.
@@ -1,5 +1,6 @@
.idea
.php_cs.cache
*.pyc
app/config/config.ini
app/config/sources/*.json
app/config/sources/*.txt
@@ -14,9 +15,9 @@ logs/github_log.txt
phpDocumentor.phar
vendor
web/assets
web/p12n
web/data.tar.gz
web/docs
web/download/.htaccess
web/download/*.tmx
web/p12n
web/TMX
@@ -5,7 +5,7 @@ Transvision is a Web application targeting the Mozilla localization community, c

The main purpose of Transvision is to provide a specialized search engine to find localized strings in Mozilla code repositories for all Mozilla products (Firefox, Thunderbird, Seamonkey) and websites (currently only www.mozilla.org is supported) via a Web interface. There are also side-features such as checks for common typographical errors for some languages, validity checks for localized access keys in the UI, or comparison views between Mozilla repository channels (Nightly/Beta/Release).

Transvision is written in PHP, the string extraction is done with the Silme library (Python) and server install/maintenance scripts are in Bash.
Transvision is written in PHP, the string extraction is done with the compare-locales library (Python) and server install/maintenance scripts are in Bash.

Transvision is available at:
https://transvision.mozfr.org
@@ -15,8 +15,6 @@ https://transvision-beta.mozfr.org

Transvision was created by Philippe Dessante, from the French Mozilla localization team.

Lead developer since version 1.0 : Pascal Chevrel (pascal AT mozilla DOT com).

## Getting Started

The Transvision team uses Git and GitHub for both development and issue tracking.
@@ -15,7 +15,7 @@
* Calls are like this:
* api/<version>/<service>/<repository>/<search type>/<source locale>/<target locale>/<url escaped search>/?optional_parameter1=foo&optional_parameter2=bar
* Example for an entity search containing bookmark:
* https://transvision.mozfr.org/api/v1/tm/release/entity/en-US/fr/bookmark/?case_sensitive=case_sensitive
* https://transvision.mozfr.org/api/v1/tm/gecko_strings/entity/en-US/fr/bookmark/?case_sensitive=case_sensitive
* (tm = translation memory service)
*
* Example for the list of locales supported for a repo:
@@ -212,7 +212,7 @@ private function isValidServiceCall($service)

break;
case 'locales':
// ex: /api/v1/locales/release/
// ex: /api/v1/locales/gecko_strings/
if (! $this->verifyEnoughParameters(3)) {
return false;
}
@@ -276,10 +276,10 @@ private function isValidServiceCall($service)
case 'suggestions':
/*
Use the same settings as 'tm'
ex: /api/v1/suggestions/release/en-US/fr/string/Home%20page/?max_results=3
ex: /api/v1/suggestions/gecko_strings/en-US/fr/string/Home%20page/?max_results=3
*/
case 'tm':
// ex: /api/v1/tm/release/en-US/fr/string/Home%20page/?max_results=3&min_quality=80
// ex: /api/v1/tm/gecko_strings/en-US/fr/string/Home%20page/?max_results=3&min_quality=80
if (! $this->verifyEnoughParameters(6)) {
return false;
}
@@ -45,21 +45,18 @@ public static function differences($tmx_source, $tmx_target, $repo, $ignored_str
{
$pattern_mismatch = [];

switch ($repo) {
case 'firefox_ios':
$patterns = [
'ios' => '/(%(?:[0-9]+\$){0,1}@)/i', // %@, but also %1$@, %2$@, etc.
];
break;
default:
$patterns = [
'dtd' => '/&([a-z0-9\.]+);/i', // &foobar;
'printf' => '/(%(?:[0-9]+\$){0,1}(?:[0-9].){0,1}(S))/i', // %1$S or %S. %1$0.S and %0.S are valid too
'properties' => '/(?<!%[0-9])\$[a-z0-9\.]+\b/i', // $BrandShortName, but not "My%1$SFeeds-%2$S.opml"
'l10njs' => '/\{\{\s*([a-z0-9_]+)\s*\}\}/iu', // {{foobar2}} Used in Loop and PDFViewer
];
break;
}
$patterns = [
'dtd' => '/&([A-Za-z0-9\.]+);/', // &foobar;
'printf' => '/(%(?:[0-9]+\$){0,1}(?:[0-9].){0,1}([sS]))/', // %1$S or %S. %1$0.S and %0.S are valid too
'properties' => '/(?<!%[0-9])\$[A-Za-z0-9\.]+\b/', // $BrandShortName, but not "My%1$SFeeds-%2$S.opml"
'l10njs' => '/\{\{\s*([A-Za-z0-9_]+)\s*\}\}/u', // {{foobar2}} Used in Loop and PDFViewer
'ios' => '/(%(?:[0-9]+\$){0,1}@)/i', // %@, but also %1$@, %2$@, etc.
];
$repo_patterns = Project::$repos_info[$repo]['variable_patterns'];

$patterns = array_filter($patterns, function($k) use ($repo_patterns) {
return in_array($k, $repo_patterns);
}, ARRAY_FILTER_USE_KEY);

foreach ($patterns as $pattern_name => $pattern) {
foreach ($tmx_source as $key => $source) {
@@ -137,6 +137,6 @@ public static function getStrings($file, $reference_locale)
*/
public static function generateStringID($file_path, $string)
{
return $file_path . ':' . hash('crc32', $string);
return $file_path . ':' . hash('md5', $string);
}
}
@@ -36,27 +36,24 @@ public static function fileForceContents($dir, $contents)
}

/**
* Return the list of files in a folder as an array.
* Hidden files starting with a dot (.svn, .htaccess...) are ignored.
* Return the list of folders in a path as an array.
* Hidden folders starting with a dot (.svn, .htaccess...) are ignored.
*
* @param string $folder the directory we want to scan
* @param array $excluded_files Files to exclude from results
* @param string $path The directory we want to scan
* @param array $excluded_folders Folders to exclude from results
*
* @return array Files in the folder
* @return array Folders in path
*/
public static function getFilenamesInFolder($folder, $excluded_files = [])
public static function getFoldersInPath($path, $excluded_folders = [])
{
/*
Here we exclude by default hidden files starting with a dot and
the . and .. symbols for directories
*/
$files = array_filter(
scandir($folder),
function ($item) {
return !Strings::startsWith($item, '.');
// We exclude by default hidden folders starting with a dot
$folders = array_filter(
scandir($path),
function ($item) use ($path) {
return is_dir("{$path}/{$item}") && ! Strings::startsWith($item, '.');
}
);

return array_diff($files, $excluded_files);
return array_diff($folders, $excluded_folders);
}
}
@@ -0,0 +1,87 @@
<?php
namespace Transvision;

use Gettext\Translations;

/**
* Po class
*
* This class is used to manipulate translation files in Gettext (.po) format.
*
* @package Transvision
*/
class Po
{
/**
*
* Loads strings from a .po file
*
* @param string $po_path Path to the .po to load
* @param string $file_name Name of the file extracted
* @param string $project_name The project this string belongs to
* @param boolean $template If I'm looking at templates
*
* @return array Array of strings as [string_id => translation]
*/
public static function getStrings($po_path, $file_name, $project_name, $template = false)
{
$translations = Translations::fromPoFile($po_path);
$strings = [];

foreach ($translations as $translation_obj) {
$translated_string = $translation_obj->getTranslation();

// Ignore fuzzy strings
if (in_array('fuzzy', $translation_obj->getFlags())) {
continue;
}

// Ignore empty (untranslated) strings
if ($translated_string == '' && ! $template) {
continue;
}

// In templates, use the original string as translation
if ($template) {
$translated_string = $translation_obj->getOriginal();
}

$string_id = self::generateStringID(
$project_name,
$file_name,
$translation_obj->getContext() . '-' . $translation_obj->getOriginal()
);
$translated_string = str_replace("'", "\\'", $translated_string);
$strings[$string_id] = $translated_string;

// Check if there are plurals, in case put them as translation of
// the only English plural form
if ($translation_obj->hasPluralTranslations()) {
$string_id = self::generateStringID(
$project_name,
$file_name,
$translation_obj->getContext() . '-' . $translation_obj->getPlural()
);
$translated_string = implode("\n", $translation_obj->getPluralTranslations());
$translated_string = str_replace("'", "\\'", $translated_string);
$strings[$string_id] = $translated_string;
}
}

return $strings;
}

/**
* Generate a unique ID for a string to store in Transvision.
*
* @param string $project_name The project this string belongs to
* @param string $file_name .po file name
* @param string $string_id String ID (context-original text)
*
* @return string unique ID such as focus_android/app.po:1dafea7725862ca854c408f0e2df9c88
*/
public static function generateStringID($project_name, $file_name, $string_id)
{
return "{$project_name}/{$file_name}:" . hash('md5', $string_id);
}
}
@@ -25,6 +25,99 @@ class Project
'calendar' => 'Lightning',
];

/*
* This array contains information about supported repositories.
*
* files: list of files to analyze during extraction
*
* git_repository: name of remote Git repository in the mozilla-l10n org
*
* git_subfolder: if localizations are in a subdirectory, e.g. if they're
* subfolders in /locales, value will be simply "locales" (no ending or
* starting /)
*
* locale_mapping: if locale codes need to be mapped (Mozilla code -> Repo code)
*
* pontoon_project: name of the project in Pontoon
*
* source_type: source type used by the project (xliff, gettext, etc.)
*
* variable_patterns: list of patterns used to check for errors in variables.
* Actual patterns (regex) are defined in the AnalyseStrings class
*
* @var array
*
*/
public static $repos_info = [
'firefox_ios' => [
'files' => [
'firefox-ios.xliff',
],
'git_repository' => 'firefoxios-l10n',
'locale_mapping' => [
'bn-IN' => 'bn',
'bn-BD' => 'bn',
'es-ES' => 'es',
'son' => 'ses',
],
'pontoon_project' => 'firefox-for-ios',
'source_type' => 'xliff',
'variable_patterns' => ['ios'],
],
'focus_android' => [
'files' => [
'app.po',
],
'git_repository' => 'focus-android-l10n',
'git_subfolder' => 'locales',
'pontoon_project' => 'focus-for-android',
'source_type' => 'gettext',
'variable_patterns' => ['l10njs', 'printf'],
],
'focus_ios' => [
'files' => [
'focus-ios.xliff',
],
'git_repository' => 'focusios-l10n',
'locale_mapping' => [
'bn-IN' => 'bn',
'bn-BD' => 'bn',
'son' => 'ses',
],
'pontoon_project' => 'focus-for-ios',
'source_type' => 'xliff',
'variable_patterns' => ['ios'],
],
'gecko_strings'=> [
'source_type' => 'mixed',
'variable_patterns' => ['dtd', 'l10njs', 'printf', 'properties'],
],
'mozilla_org'=> [
'git_repository' => 'www.mozilla.org',
'pontoon_project' => 'mozillaorg',
'source_type' => 'dotlang',
],
];

/*
Since Project is used statically, not as an object, it would be too
expensive to generate the list of repos dinamically from $repos_info.
*/
public static $repos_lists = [
// Desktop products
'desktop' => [
'gecko_strings',
],
// Products using Git
'git' => [
'firefox_ios', 'focus_android', 'focus_ios', 'mozilla_org',
],
// Products using free text search on Pontoon
'text_search' => [
'firefox_ios', 'focus_android', 'focus_ios', 'mozilla_org',
],
];

/**
* Create a list of all supported repositories.
*
@@ -56,7 +149,10 @@ public static function getSupportedRepositories()
*/
public static function getRepositories()
{
return array_keys(self::getSupportedRepositories());
$repositories = array_keys(self::getSupportedRepositories());
sort($repositories);

return $repositories;
}

/**
@@ -78,10 +174,7 @@ public static function getRepositoriesNames()
*/
public static function getDesktopRepositories()
{
return array_diff(
self::getRepositories(),
['mozilla_org', 'firefox_ios']
);
return self::$repos_lists['desktop'];
}

/**
@@ -196,25 +289,17 @@ public static function getLocaleInContext($locale, $context)
'sr-Latn' => 'sr',
];

// Firefox for iOS
$locale_mappings['firefox_ios'] = [
'bn-IN' => 'bn',
'bn-BD' => 'bn',
'es-AR' => 'es',
'es-ES' => 'es',
'son' => 'ses',
];

// For other contexts use the same as Bugzilla
$locale_mappings['other'] = $locale_mappings['bugzilla'];

// Fallback to 'other' if context doesn't exist in $locale_mappings
$context = array_key_exists($context, $locale_mappings)
? $context
: 'other';
// Fall back to Bugzilla if there are no mappings for the requested context
if (isset(self::$repos_info[$context]['locale_mapping'])) {
$mapping = self::$repos_info[$context]['locale_mapping'];
} elseif (isset($locale_mappings[$context])) {
$mapping = $locale_mappings[$context];
} else {
$mapping = $locale_mappings['bugzilla'];
}

$locale = array_key_exists($locale, $locale_mappings[$context])
? $locale_mappings[$context][$locale]
$locale = array_key_exists($locale, $mapping)
? $mapping[$locale]
: $locale;

return $locale;

0 comments on commit d1dbaa3

Please sign in to comment.