Permalink
Browse files

Implement cross-channel localization support, add Focus for iOS and A…

…ndroid (#895)
  • Loading branch information...
flodolo committed Oct 11, 2017
1 parent 28f2954 commit d1dbaa349333adb7f1d23195099d3b4dfeadc7e4
Showing with 1,246 additions and 989 deletions.
  1. +2 −1 .gitignore
  2. +1 −3 README.md
  3. +4 −4 app/classes/Transvision/API.php
  4. +12 −15 app/classes/Transvision/AnalyseStrings.php
  5. +1 −1 app/classes/Transvision/Dotlang.php
  6. +12 −15 app/classes/Transvision/Files.php
  7. +87 −0 app/classes/Transvision/Po.php
  8. +108 −23 app/classes/Transvision/Project.php
  9. +2 −2 app/classes/Transvision/Search.php
  10. +12 −27 app/classes/Transvision/ShowResults.php
  11. +50 −111 app/classes/Transvision/VersionControl.php
  12. +6 −13 app/classes/Transvision/Xliff.php
  13. +1 −1 app/config/config.ini-dist
  14. +1 −1 app/config/config.ini-travis
  15. +0 −5 app/controllers/channelcomparison.php
  16. +1 −1 app/controllers/mainsearch.php
  17. +1 −1 app/controllers/onestring.php
  18. +0 −7 app/inc/dispatcher.php
  19. +1 −0 app/inc/init.php
  20. +1 −1 app/inc/search_options.php
  21. +0 −1 app/inc/urls.php
  22. +3 −3 app/models/changelog_data.php
  23. +0 −69 app/models/channelcomparison.php
  24. +0 −10 app/models/empty_strings.php
  25. +4 −1 app/models/health_status.php
  26. +9 −17 app/scripts/bash_variables.sh
  27. +4 −7 app/scripts/clean_data.py
  28. +12 −8 app/scripts/generate_sources
  29. +56 −91 app/scripts/glossaire.sh
  30. +60 −135 app/scripts/setup.sh
  31. +41 −0 app/scripts/tmx/env_setup.py
  32. +69 −0 app/scripts/tmx/tmx_gettext
  33. +2 −2 app/scripts/{ → tmx}/tmx_mozillaorg
  34. +26 −26 app/scripts/{ → tmx}/tmx_products.py
  35. +22 −19 app/scripts/{ → tmx}/tmx_xliff
  36. +0 −104 app/views/channelcomparison.php
  37. +1 −1 app/views/credits.php
  38. +3 −3 app/views/productization.php
  39. +3 −12 app/views/results_entities.php
  40. +0 −1 app/views/search_form.php
  41. +1 −2 app/views/templates/base.php
  42. +2 −1 composer.json
  43. +13 −14 tests/functional/api.php
  44. +6 −7 tests/functional/pages.php
  45. 0 tests/testfiles/TMX/en-US/{cache_en-US_central.php → cache_en-US_gecko_strings.php}
  46. 0 tests/testfiles/TMX/fr/{cache_fr_central.php → cache_fr_gecko_strings.php}
  47. 0 tests/testfiles/config/{central.txt → sources/gecko_strings.txt}
  48. 0 tests/testfiles/config/{ → sources}/mozilla_org.txt
  49. +10 −0 tests/testfiles/config/sources/supported_repositories.json
  50. 0 tests/testfiles/config/{ → sources}/tools.json
  51. +2 −10 tests/testfiles/config/supported_repositories.json
  52. +114 −0 tests/testfiles/po/app.pot
  53. +28 −0 tests/testfiles/po/ga_app.po
  54. +137 −0 tests/testfiles/po/it_app.po
  55. +33 −33 tests/units/Transvision/API.php
  56. +25 −17 tests/units/Transvision/AnalyseStrings.php
  57. +1 −1 tests/units/Transvision/Consistency.php
  58. +1 −1 tests/units/Transvision/Dotlang.php
  59. +95 −0 tests/units/Transvision/Po.php
  60. +14 −16 tests/units/Transvision/Project.php
  61. +5 −5 tests/units/Transvision/Search.php
  62. +40 −32 tests/units/Transvision/ShowResults.php
  63. +9 −9 tests/units/Transvision/Utils.php
  64. +56 −51 tests/units/Transvision/VersionControl.php
  65. +6 −6 tests/units/Transvision/Xliff.php
  66. +1 −1 tests/units/bootstrap.php
  67. +26 −3 web/github_hook.php
  68. +1 −1 web/js/base.js
  69. +2 −37 web/style/transvision.css
View
@@ -1,5 +1,6 @@
.idea
.php_cs.cache
*.pyc
app/config/config.ini
app/config/sources/*.json
app/config/sources/*.txt
@@ -14,9 +15,9 @@ logs/github_log.txt
phpDocumentor.phar
vendor
web/assets
web/p12n
web/data.tar.gz
web/docs
web/download/.htaccess
web/download/*.tmx
web/p12n
web/TMX
View
@@ -5,7 +5,7 @@ Transvision is a Web application targeting the Mozilla localization community, c
The main purpose of Transvision is to provide a specialized search engine to find localized strings in Mozilla code repositories for all Mozilla products (Firefox, Thunderbird, Seamonkey) and websites (currently only www.mozilla.org is supported) via a Web interface. There are also side-features such as checks for common typographical errors for some languages, validity checks for localized access keys in the UI, or comparison views between Mozilla repository channels (Nightly/Beta/Release).
Transvision is written in PHP, the string extraction is done with the Silme library (Python) and server install/maintenance scripts are in Bash.
Transvision is written in PHP, the string extraction is done with the compare-locales library (Python) and server install/maintenance scripts are in Bash.
Transvision is available at:
https://transvision.mozfr.org
@@ -15,8 +15,6 @@ https://transvision-beta.mozfr.org
Transvision was created by Philippe Dessante, from the French Mozilla localization team.
Lead developer since version 1.0 : Pascal Chevrel (pascal AT mozilla DOT com).
## Getting Started
The Transvision team uses Git and GitHub for both development and issue tracking.
@@ -15,7 +15,7 @@
* Calls are like this:
* api/<version>/<service>/<repository>/<search type>/<source locale>/<target locale>/<url escaped search>/?optional_parameter1=foo&optional_parameter2=bar
* Example for an entity search containing bookmark:
* https://transvision.mozfr.org/api/v1/tm/release/entity/en-US/fr/bookmark/?case_sensitive=case_sensitive
* https://transvision.mozfr.org/api/v1/tm/gecko_strings/entity/en-US/fr/bookmark/?case_sensitive=case_sensitive
* (tm = translation memory service)
*
* Example for the list of locales supported for a repo:
@@ -212,7 +212,7 @@ private function isValidServiceCall($service)
break;
case 'locales':
// ex: /api/v1/locales/release/
// ex: /api/v1/locales/gecko_strings/
if (! $this->verifyEnoughParameters(3)) {
return false;
}
@@ -276,10 +276,10 @@ private function isValidServiceCall($service)
case 'suggestions':
/*
Use the same settings as 'tm'
ex: /api/v1/suggestions/release/en-US/fr/string/Home%20page/?max_results=3
ex: /api/v1/suggestions/gecko_strings/en-US/fr/string/Home%20page/?max_results=3
*/
case 'tm':
// ex: /api/v1/tm/release/en-US/fr/string/Home%20page/?max_results=3&min_quality=80
// ex: /api/v1/tm/gecko_strings/en-US/fr/string/Home%20page/?max_results=3&min_quality=80
if (! $this->verifyEnoughParameters(6)) {
return false;
}
@@ -45,21 +45,18 @@ public static function differences($tmx_source, $tmx_target, $repo, $ignored_str
{
$pattern_mismatch = [];
switch ($repo) {
case 'firefox_ios':
$patterns = [
'ios' => '/(%(?:[0-9]+\$){0,1}@)/i', // %@, but also %1$@, %2$@, etc.
];
break;
default:
$patterns = [
'dtd' => '/&([a-z0-9\.]+);/i', // &foobar;
'printf' => '/(%(?:[0-9]+\$){0,1}(?:[0-9].){0,1}(S))/i', // %1$S or %S. %1$0.S and %0.S are valid too
'properties' => '/(?<!%[0-9])\$[a-z0-9\.]+\b/i', // $BrandShortName, but not "My%1$SFeeds-%2$S.opml"
'l10njs' => '/\{\{\s*([a-z0-9_]+)\s*\}\}/iu', // {{foobar2}} Used in Loop and PDFViewer
];
break;
}
$patterns = [
'dtd' => '/&([A-Za-z0-9\.]+);/', // &foobar;
'printf' => '/(%(?:[0-9]+\$){0,1}(?:[0-9].){0,1}([sS]))/', // %1$S or %S. %1$0.S and %0.S are valid too
'properties' => '/(?<!%[0-9])\$[A-Za-z0-9\.]+\b/', // $BrandShortName, but not "My%1$SFeeds-%2$S.opml"
'l10njs' => '/\{\{\s*([A-Za-z0-9_]+)\s*\}\}/u', // {{foobar2}} Used in Loop and PDFViewer
'ios' => '/(%(?:[0-9]+\$){0,1}@)/i', // %@, but also %1$@, %2$@, etc.
];
$repo_patterns = Project::$repos_info[$repo]['variable_patterns'];
$patterns = array_filter($patterns, function($k) use ($repo_patterns) {
return in_array($k, $repo_patterns);
}, ARRAY_FILTER_USE_KEY);
foreach ($patterns as $pattern_name => $pattern) {
foreach ($tmx_source as $key => $source) {
@@ -137,6 +137,6 @@ public static function getStrings($file, $reference_locale)
*/
public static function generateStringID($file_path, $string)
{
return $file_path . ':' . hash('crc32', $string);
return $file_path . ':' . hash('md5', $string);
}
}
@@ -36,27 +36,24 @@ public static function fileForceContents($dir, $contents)
}
/**
* Return the list of files in a folder as an array.
* Hidden files starting with a dot (.svn, .htaccess...) are ignored.
* Return the list of folders in a path as an array.
* Hidden folders starting with a dot (.svn, .htaccess...) are ignored.
*
* @param string $folder the directory we want to scan
* @param array $excluded_files Files to exclude from results
* @param string $path The directory we want to scan
* @param array $excluded_folders Folders to exclude from results
*
* @return array Files in the folder
* @return array Folders in path
*/
public static function getFilenamesInFolder($folder, $excluded_files = [])
public static function getFoldersInPath($path, $excluded_folders = [])
{
/*
Here we exclude by default hidden files starting with a dot and
the . and .. symbols for directories
*/
$files = array_filter(
scandir($folder),
function ($item) {
return !Strings::startsWith($item, '.');
// We exclude by default hidden folders starting with a dot
$folders = array_filter(
scandir($path),
function ($item) use ($path) {
return is_dir("{$path}/{$item}") && ! Strings::startsWith($item, '.');
}
);
return array_diff($files, $excluded_files);
return array_diff($folders, $excluded_folders);
}
}
@@ -0,0 +1,87 @@
<?php
namespace Transvision;
use Gettext\Translations;
/**
* Po class
*
* This class is used to manipulate translation files in Gettext (.po) format.
*
* @package Transvision
*/
class Po
{
/**
*
* Loads strings from a .po file
*
* @param string $po_path Path to the .po to load
* @param string $file_name Name of the file extracted
* @param string $project_name The project this string belongs to
* @param boolean $template If I'm looking at templates
*
* @return array Array of strings as [string_id => translation]
*/
public static function getStrings($po_path, $file_name, $project_name, $template = false)
{
$translations = Translations::fromPoFile($po_path);
$strings = [];
foreach ($translations as $translation_obj) {
$translated_string = $translation_obj->getTranslation();
// Ignore fuzzy strings
if (in_array('fuzzy', $translation_obj->getFlags())) {
continue;
}
// Ignore empty (untranslated) strings
if ($translated_string == '' && ! $template) {
continue;
}
// In templates, use the original string as translation
if ($template) {
$translated_string = $translation_obj->getOriginal();
}
$string_id = self::generateStringID(
$project_name,
$file_name,
$translation_obj->getContext() . '-' . $translation_obj->getOriginal()
);
$translated_string = str_replace("'", "\\'", $translated_string);
$strings[$string_id] = $translated_string;
// Check if there are plurals, in case put them as translation of
// the only English plural form
if ($translation_obj->hasPluralTranslations()) {
$string_id = self::generateStringID(
$project_name,
$file_name,
$translation_obj->getContext() . '-' . $translation_obj->getPlural()
);
$translated_string = implode("\n", $translation_obj->getPluralTranslations());
$translated_string = str_replace("'", "\\'", $translated_string);
$strings[$string_id] = $translated_string;
}
}
return $strings;
}
/**
* Generate a unique ID for a string to store in Transvision.
*
* @param string $project_name The project this string belongs to
* @param string $file_name .po file name
* @param string $string_id String ID (context-original text)
*
* @return string unique ID such as focus_android/app.po:1dafea7725862ca854c408f0e2df9c88
*/
public static function generateStringID($project_name, $file_name, $string_id)
{
return "{$project_name}/{$file_name}:" . hash('md5', $string_id);
}
}
@@ -25,6 +25,99 @@ class Project
'calendar' => 'Lightning',
];
/*
* This array contains information about supported repositories.
*
* files: list of files to analyze during extraction
*
* git_repository: name of remote Git repository in the mozilla-l10n org
*
* git_subfolder: if localizations are in a subdirectory, e.g. if they're
* subfolders in /locales, value will be simply "locales" (no ending or
* starting /)
*
* locale_mapping: if locale codes need to be mapped (Mozilla code -> Repo code)
*
* pontoon_project: name of the project in Pontoon
*
* source_type: source type used by the project (xliff, gettext, etc.)
*
* variable_patterns: list of patterns used to check for errors in variables.
* Actual patterns (regex) are defined in the AnalyseStrings class
*
* @var array
*
*/
public static $repos_info = [
'firefox_ios' => [
'files' => [
'firefox-ios.xliff',
],
'git_repository' => 'firefoxios-l10n',
'locale_mapping' => [
'bn-IN' => 'bn',
'bn-BD' => 'bn',
'es-ES' => 'es',
'son' => 'ses',
],
'pontoon_project' => 'firefox-for-ios',
'source_type' => 'xliff',
'variable_patterns' => ['ios'],
],
'focus_android' => [
'files' => [
'app.po',
],
'git_repository' => 'focus-android-l10n',
'git_subfolder' => 'locales',
'pontoon_project' => 'focus-for-android',
'source_type' => 'gettext',
'variable_patterns' => ['l10njs', 'printf'],
],
'focus_ios' => [
'files' => [
'focus-ios.xliff',
],
'git_repository' => 'focusios-l10n',
'locale_mapping' => [
'bn-IN' => 'bn',
'bn-BD' => 'bn',
'son' => 'ses',
],
'pontoon_project' => 'focus-for-ios',
'source_type' => 'xliff',
'variable_patterns' => ['ios'],
],
'gecko_strings'=> [
'source_type' => 'mixed',
'variable_patterns' => ['dtd', 'l10njs', 'printf', 'properties'],
],
'mozilla_org'=> [
'git_repository' => 'www.mozilla.org',
'pontoon_project' => 'mozillaorg',
'source_type' => 'dotlang',
],
];
/*
Since Project is used statically, not as an object, it would be too
expensive to generate the list of repos dinamically from $repos_info.
*/
public static $repos_lists = [
// Desktop products
'desktop' => [
'gecko_strings',
],
// Products using Git
'git' => [
'firefox_ios', 'focus_android', 'focus_ios', 'mozilla_org',
],
// Products using free text search on Pontoon
'text_search' => [
'firefox_ios', 'focus_android', 'focus_ios', 'mozilla_org',
],
];
/**
* Create a list of all supported repositories.
*
@@ -56,7 +149,10 @@ public static function getSupportedRepositories()
*/
public static function getRepositories()
{
return array_keys(self::getSupportedRepositories());
$repositories = array_keys(self::getSupportedRepositories());
sort($repositories);
return $repositories;
}
/**
@@ -78,10 +174,7 @@ public static function getRepositoriesNames()
*/
public static function getDesktopRepositories()
{
return array_diff(
self::getRepositories(),
['mozilla_org', 'firefox_ios']
);
return self::$repos_lists['desktop'];
}
/**
@@ -196,25 +289,17 @@ public static function getLocaleInContext($locale, $context)
'sr-Latn' => 'sr',
];
// Firefox for iOS
$locale_mappings['firefox_ios'] = [
'bn-IN' => 'bn',
'bn-BD' => 'bn',
'es-AR' => 'es',
'es-ES' => 'es',
'son' => 'ses',
];
// For other contexts use the same as Bugzilla
$locale_mappings['other'] = $locale_mappings['bugzilla'];
// Fallback to 'other' if context doesn't exist in $locale_mappings
$context = array_key_exists($context, $locale_mappings)
? $context
: 'other';
// Fall back to Bugzilla if there are no mappings for the requested context
if (isset(self::$repos_info[$context]['locale_mapping'])) {
$mapping = self::$repos_info[$context]['locale_mapping'];
} elseif (isset($locale_mappings[$context])) {
$mapping = $locale_mappings[$context];
} else {
$mapping = $locale_mappings['bugzilla'];
}
$locale = array_key_exists($locale, $locale_mappings[$context])
? $locale_mappings[$context][$locale]
$locale = array_key_exists($locale, $mapping)
? $mapping[$locale]
: $locale;
return $locale;
Oops, something went wrong.

0 comments on commit d1dbaa3

Please sign in to comment.