Implement cross-channel localization support, add Focus for iOS and Android (#895)

Author: flodolo

.gitignore

@@ -1,5 +1,6 @@
 .idea
 .php_cs.cache
+*.pyc
 app/config/config.ini
 app/config/sources/*.json
 app/config/sources/*.txt
@@ -14,9 +15,9 @@ logs/github_log.txt
 phpDocumentor.phar
 vendor
 web/assets
-web/p12n
 web/data.tar.gz
 web/docs
 web/download/.htaccess
 web/download/*.tmx
+web/p12n
 web/TMX

README.md

@@ -5,7 +5,7 @@ Transvision is a Web application targeting the Mozilla localization community, c
 
 The main purpose of Transvision is to provide a specialized search engine to find localized strings in Mozilla code repositories for all Mozilla products (Firefox, Thunderbird, Seamonkey) and websites (currently only www.mozilla.org is supported) via a Web interface. There are also side-features such as checks for common typographical errors for some languages, validity checks for localized access keys in the UI, or comparison views between Mozilla repository channels (Nightly/Beta/Release).
 
-Transvision is written in PHP, the string extraction is done with the Silme library (Python) and server install/maintenance scripts are in Bash.
+Transvision is written in PHP, the string extraction is done with the compare-locales library (Python) and server install/maintenance scripts are in Bash.
 
 Transvision is available at:
 https://transvision.mozfr.org
@@ -15,8 +15,6 @@ https://transvision-beta.mozfr.org
 
 Transvision was created by Philippe Dessante, from the French Mozilla localization team.
 
-Lead developer since version 1.0 : Pascal Chevrel (pascal AT mozilla DOT com).
-
 ## Getting Started
 
 The Transvision team uses Git and GitHub for both development and issue tracking.

app/classes/Transvision/API.php

@@ -15,7 +15,7 @@
  * Calls are like this:
  * api/<version>/<service>/<repository>/<search type>/<source locale>/<target locale>/<url escaped search>/?optional_parameter1=foo&optional_parameter2=bar
  * Example for an entity search containing bookmark:
- * https://transvision.mozfr.org/api/v1/tm/release/entity/en-US/fr/bookmark/?case_sensitive=case_sensitive
+ * https://transvision.mozfr.org/api/v1/tm/gecko_strings/entity/en-US/fr/bookmark/?case_sensitive=case_sensitive
  * (tm = translation memory service)
  *
  * Example for the list of locales supported for a repo:
@@ -212,7 +212,7 @@ private function isValidServiceCall($service)
 
                 break;
             case 'locales':
-            // ex: /api/v1/locales/release/
+            // ex: /api/v1/locales/gecko_strings/
                 if (! $this->verifyEnoughParameters(3)) {
                     return false;
                 }
@@ -276,10 +276,10 @@ private function isValidServiceCall($service)
             case 'suggestions':
             /*
                 Use the same settings as 'tm'
-                ex: /api/v1/suggestions/release/en-US/fr/string/Home%20page/?max_results=3
+                ex: /api/v1/suggestions/gecko_strings/en-US/fr/string/Home%20page/?max_results=3
             */
             case 'tm':
-            // ex: /api/v1/tm/release/en-US/fr/string/Home%20page/?max_results=3&min_quality=80
+            // ex: /api/v1/tm/gecko_strings/en-US/fr/string/Home%20page/?max_results=3&min_quality=80
                 if (! $this->verifyEnoughParameters(6)) {
                     return false;
                 }

app/classes/Transvision/AnalyseStrings.php

@@ -45,21 +45,18 @@ public static function differences($tmx_source, $tmx_target, $repo, $ignored_str
     {
         $pattern_mismatch = [];
 
-        switch ($repo) {
-            case 'firefox_ios':
-                $patterns = [
-                    'ios' => '/(%(?:[0-9]+\$){0,1}@)/i', // %@, but also %1$@, %2$@, etc.
-                ];
-                break;
-            default:
-                $patterns = [
-                    'dtd'        => '/&([a-z0-9\.]+);/i',                        // &foobar;
-                    'printf'     => '/(%(?:[0-9]+\$){0,1}(?:[0-9].){0,1}(S))/i', // %1$S or %S. %1$0.S and %0.S are valid too
-                    'properties' => '/(?<!%[0-9])\$[a-z0-9\.]+\b/i',             // $BrandShortName, but not "My%1$SFeeds-%2$S.opml"
-                    'l10njs'     => '/\{\{\s*([a-z0-9_]+)\s*\}\}/iu',            // {{foobar2}} Used in Loop and PDFViewer
-                ];
-                break;
-        }
+        $patterns = [
+            'dtd'        => '/&([A-Za-z0-9\.]+);/',                        // &foobar;
+            'printf'     => '/(%(?:[0-9]+\$){0,1}(?:[0-9].){0,1}([sS]))/', // %1$S or %S. %1$0.S and %0.S are valid too
+            'properties' => '/(?<!%[0-9])\$[A-Za-z0-9\.]+\b/',             // $BrandShortName, but not "My%1$SFeeds-%2$S.opml"
+            'l10njs'     => '/\{\{\s*([A-Za-z0-9_]+)\s*\}\}/u',            // {{foobar2}} Used in Loop and PDFViewer
+            'ios'        => '/(%(?:[0-9]+\$){0,1}@)/i',                    // %@, but also %1$@, %2$@, etc.
+        ];
+        $repo_patterns = Project::$repos_info[$repo]['variable_patterns'];
+
+        $patterns = array_filter($patterns, function($k) use ($repo_patterns) {
+            return in_array($k, $repo_patterns);
+        }, ARRAY_FILTER_USE_KEY);
 
         foreach ($patterns as $pattern_name => $pattern) {
             foreach ($tmx_source as $key => $source) {

app/classes/Transvision/Dotlang.php

@@ -137,6 +137,6 @@ public static function getStrings($file, $reference_locale)
      */
     public static function generateStringID($file_path, $string)
     {
-        return $file_path . ':' . hash('crc32', $string);
+        return $file_path . ':' . hash('md5', $string);
     }
 }

app/classes/Transvision/Files.php

@@ -36,27 +36,24 @@ public static function fileForceContents($dir, $contents)
     }
 
     /**
-     * Return the list of files in a folder as an array.
-     * Hidden files starting with a dot (.svn, .htaccess...) are ignored.
+     * Return the list of folders in a path as an array.
+     * Hidden folders starting with a dot (.svn, .htaccess...) are ignored.
      *
-     * @param string $folder         the directory we want to scan
-     * @param array  $excluded_files Files to exclude from results
+     * @param string $path             The directory we want to scan
+     * @param array  $excluded_folders Folders to exclude from results
      *
-     * @return array Files in the folder
+     * @return array Folders in path
      */
-    public static function getFilenamesInFolder($folder, $excluded_files = [])
+    public static function getFoldersInPath($path, $excluded_folders = [])
     {
-        /*
-            Here we exclude by default hidden files starting with a dot and
-            the . and .. symbols for directories
-        */
-        $files = array_filter(
-            scandir($folder),
-            function ($item) {
-                return !Strings::startsWith($item, '.');
+        // We exclude by default hidden folders starting with a dot
+        $folders = array_filter(
+            scandir($path),
+            function ($item) use ($path) {
+                return is_dir("{$path}/{$item}") && ! Strings::startsWith($item, '.');
             }
         );
 
-        return array_diff($files, $excluded_files);
+        return array_diff($folders, $excluded_folders);
     }
 }

app/classes/Transvision/Po.php

@@ -0,0 +1,87 @@
+<?php
+namespace Transvision;
+
+use Gettext\Translations;
+
+/**
+ * Po class
+ *
+ * This class is used to manipulate translation files in Gettext (.po) format.
+ *
+ * @package Transvision
+ */
+class Po
+{
+    /**
+     *
+     * Loads strings from a .po file
+     *
+     * @param string  $po_path      Path to the .po to load
+     * @param string  $file_name    Name of the file extracted
+     * @param string  $project_name The project this string belongs to
+     * @param boolean $template     If I'm looking at templates
+     *
+     * @return array Array of strings as [string_id => translation]
+     */
+    public static function getStrings($po_path, $file_name, $project_name, $template = false)
+    {
+        $translations = Translations::fromPoFile($po_path);
+        $strings = [];
+
+        foreach ($translations as $translation_obj) {
+            $translated_string = $translation_obj->getTranslation();
+
+            // Ignore fuzzy strings
+            if (in_array('fuzzy', $translation_obj->getFlags())) {
+                continue;
+            }
+
+            // Ignore empty (untranslated) strings
+            if ($translated_string == '' && ! $template) {
+                continue;
+            }
+
+            // In templates, use the original string as translation
+            if ($template) {
+                $translated_string = $translation_obj->getOriginal();
+            }
+
+            $string_id = self::generateStringID(
+                $project_name,
+                $file_name,
+                $translation_obj->getContext() . '-' . $translation_obj->getOriginal()
+            );
+            $translated_string = str_replace("'", "\\'", $translated_string);
+            $strings[$string_id] = $translated_string;
+
+            // Check if there are plurals, in case put them as translation of
+            // the only English plural form
+            if ($translation_obj->hasPluralTranslations()) {
+                $string_id = self::generateStringID(
+                    $project_name,
+                    $file_name,
+                    $translation_obj->getContext() . '-' . $translation_obj->getPlural()
+                );
+                $translated_string = implode("\n", $translation_obj->getPluralTranslations());
+                $translated_string = str_replace("'", "\\'", $translated_string);
+                $strings[$string_id] = $translated_string;
+            }
+        }
+
+        return $strings;
+    }
+
+    /**
+     * Generate a unique ID for a string to store in Transvision.
+     *
+     * @param string $project_name The project this string belongs to
+     * @param string $file_name    .po file name
+     * @param string $string_id    String ID (context-original text)
+     *
+     * @return string unique ID such as focus_android/app.po:1dafea7725862ca854c408f0e2df9c88
+     */
+    public static function generateStringID($project_name, $file_name, $string_id)
+    {
+        return "{$project_name}/{$file_name}:" . hash('md5', $string_id);
+    }
+}

app/classes/Transvision/Project.php

@@ -25,6 +25,99 @@ class Project
         'calendar' => 'Lightning',
     ];
 
+    /*
+     * This array contains information about supported repositories.
+     *
+     * files: list of files to analyze during extraction
+     *
+     * git_repository: name of remote Git repository in the mozilla-l10n org
+     *
+     * git_subfolder: if localizations are in a subdirectory, e.g. if they're
+     * subfolders in /locales, value will be simply "locales" (no ending or
+     * starting /)
+     *
+     * locale_mapping: if locale codes need to be mapped (Mozilla code -> Repo code)
+     *
+     * pontoon_project: name of the project in Pontoon
+     *
+     * source_type: source type used by the project (xliff, gettext, etc.)
+     *
+     * variable_patterns: list of patterns used to check for errors in variables.
+     * Actual patterns (regex) are defined in the AnalyseStrings class
+     *
+     * @var array
+     *
+     */
+    public static $repos_info = [
+        'firefox_ios' => [
+            'files'             => [
+                'firefox-ios.xliff',
+            ],
+            'git_repository'    => 'firefoxios-l10n',
+            'locale_mapping'    => [
+                'bn-IN' => 'bn',
+                'bn-BD' => 'bn',
+                'es-ES' => 'es',
+                'son'   => 'ses',
+            ],
+            'pontoon_project'   => 'firefox-for-ios',
+            'source_type'       => 'xliff',
+            'variable_patterns' => ['ios'],
+        ],
+        'focus_android' => [
+            'files'             => [
+                'app.po',
+            ],
+            'git_repository'    => 'focus-android-l10n',
+            'git_subfolder'     => 'locales',
+            'pontoon_project'   => 'focus-for-android',
+            'source_type'       => 'gettext',
+            'variable_patterns' => ['l10njs', 'printf'],
+        ],
+        'focus_ios' => [
+            'files'             => [
+                'focus-ios.xliff',
+            ],
+            'git_repository'    => 'focusios-l10n',
+            'locale_mapping'    => [
+                'bn-IN' => 'bn',
+                'bn-BD' => 'bn',
+                'son'   => 'ses',
+            ],
+            'pontoon_project'   => 'focus-for-ios',
+            'source_type'       => 'xliff',
+            'variable_patterns' => ['ios'],
+        ],
+        'gecko_strings'=> [
+            'source_type'       => 'mixed',
+            'variable_patterns' => ['dtd', 'l10njs', 'printf', 'properties'],
+        ],
+        'mozilla_org'=> [
+            'git_repository'    => 'www.mozilla.org',
+            'pontoon_project'   => 'mozillaorg',
+            'source_type'       => 'dotlang',
+        ],
+    ];
+
+    /*
+        Since Project is used statically, not as an object, it would be too
+        expensive to generate the list of repos dinamically from $repos_info.
+    */
+    public static $repos_lists = [
+        // Desktop products
+        'desktop'     => [
+            'gecko_strings',
+        ],
+        // Products using Git
+        'git' => [
+            'firefox_ios', 'focus_android', 'focus_ios', 'mozilla_org',
+        ],
+        // Products using free text search on Pontoon
+        'text_search' => [
+            'firefox_ios', 'focus_android', 'focus_ios', 'mozilla_org',
+        ],
+    ];
+
     /**
      * Create a list of all supported repositories.
      *
@@ -56,7 +149,10 @@ public static function getSupportedRepositories()
      */
     public static function getRepositories()
     {
-        return array_keys(self::getSupportedRepositories());
+        $repositories = array_keys(self::getSupportedRepositories());
+        sort($repositories);
+
+        return $repositories;
     }
 
     /**
@@ -78,10 +174,7 @@ public static function getRepositoriesNames()
      */
     public static function getDesktopRepositories()
     {
-        return array_diff(
-            self::getRepositories(),
-            ['mozilla_org', 'firefox_ios']
-        );
+        return self::$repos_lists['desktop'];
     }
 
     /**
@@ -196,25 +289,17 @@ public static function getLocaleInContext($locale, $context)
             'sr-Latn' => 'sr',
         ];
 
-        // Firefox for iOS
-        $locale_mappings['firefox_ios'] = [
-            'bn-IN' => 'bn',
-            'bn-BD' => 'bn',
-            'es-AR' => 'es',
-            'es-ES' => 'es',
-            'son'   => 'ses',
-        ];
-
-        // For other contexts use the same as Bugzilla
-        $locale_mappings['other'] = $locale_mappings['bugzilla'];
-
-        // Fallback to 'other' if context doesn't exist in $locale_mappings
-        $context = array_key_exists($context, $locale_mappings)
-                   ? $context
-                   : 'other';
+        // Fall back to Bugzilla if there are no mappings for the requested context
+        if (isset(self::$repos_info[$context]['locale_mapping'])) {
+            $mapping = self::$repos_info[$context]['locale_mapping'];
+        } elseif (isset($locale_mappings[$context])) {
+            $mapping = $locale_mappings[$context];
+        } else {
+            $mapping = $locale_mappings['bugzilla'];
+        }
 
-        $locale = array_key_exists($locale, $locale_mappings[$context])
-                  ? $locale_mappings[$context][$locale]
+        $locale = array_key_exists($locale, $mapping)
+                  ? $mapping[$locale]
                   : $locale;
 
         return $locale;