Issue mozfr#392: Translation memory API, query all repositories

- add a new alias for all repos called 'global' - add Project::getRepositoriesLocale('ab-CD'), returns all the repositories available for a locale - new public api/v1/repositories/<locale code>/ JSON API - 'global' search added to translation memory API + general search (strings and entities) API - Max results is now 500 for the API - dupes are removed from the translation memory API - update Project:getLocaleInContext() to have mappings for Spanishes - add new unit and functional tests
pascalchevrel · Oct 14, 2015 · a253252 · a253252
1 parent ed8bc1a
commit a253252
Show file tree

Hide file tree

Showing 12 changed files with 187 additions and 50 deletions.
diff --git a/app/classes/Transvision/API.php b/app/classes/Transvision/API.php
@@ -45,7 +45,7 @@ class API
     public $error;
     public $logging = true;
     public $logger;
-
+    public $valid_repositories;
     /**
      * The constructor analyzes the URL to extract its parameters
      *
@@ -71,6 +71,8 @@ public function __construct($url)
         $this->extra_parameters = isset($url['query'])
             ? $this->getExtraParameters($url['query'])
             : [];
+
+        $this->valid_repositories = array_merge(Project::getRepositories(), ['global']);
     }
 
     /**
@@ -227,7 +229,7 @@ private function isValidServiceCall($service)
                     return false;
                 }
 
-                if (! $this->verifyRepositoryExists($this->parameters[3])) {
+                if (! $this->verifyRepositoryExists($this->parameters[3], true)) {
                     return false;
                 }
 
@@ -261,7 +263,26 @@ private function isValidServiceCall($service)
                 break;
             case 'repositories':
                 // ex: api/repositories/
-                // Generated from Project class, no user-defined variables = nothing to check
+                // ex: api/repositories/fr/
+                // Generated from Project class
+                // There is one optional parameter, a locale code
+                if (isset($this->parameters[2])) {
+                    $match = false;
+
+                    foreach (Project::getRepositories() as $repository) {
+                        if ($this->verifyLocaleExists($this->parameters[2], $repository)) {
+                            $match = true;
+                            break;
+                        }
+                    }
+
+                    if (! $match) {
+                        $this->log("The locale queried ({$this->parameters[2]}) is not supported");
+
+                        return false;
+                    }
+                }
+
                 break;
             case 'versions':
                 // ex: api/versions/
@@ -302,11 +323,17 @@ private function verifyEnoughParameters($number)
      * Check that the repository asked for is one we support
      *
      * @param  string  $repository Name of the repository
+     * @param  boolean $alias      Do we allow aliases for repository names,
+     *                             ex: 'global', to query all repositories. Default to False
      * @return boolean True if we support this repository, False if we don't
      */
-    private function verifyRepositoryExists($repository)
+    private function verifyRepositoryExists($repository, $alias = false)
     {
-        if (! in_array($repository, Project::getRepositories())) {
+        if (! in_array($repository, $this->valid_repositories)) {
+            $this->valid_repositories = $alias
+                ? array_merge(Project::getRepositories(), ['global'])
+                : Project::getRepositories();
+
             $this->log("The repo queried ({$repository}) doesn't exist.");
 
             return false;
@@ -324,6 +351,12 @@ private function verifyRepositoryExists($repository)
      */
     private function verifyLocaleExists($locale, $repository)
     {
+        if ($repository == 'global') {
+            if (! empty(Project::getLocaleRepositories($locale))) {
+                return true;
+            }
+        }
+
         if (! in_array($locale, Project::getRepositoryLocales($repository))) {
             $this->log("The locale queried ({$locale}) is not "
                        . "available for the repository ({$repository}).");

diff --git a/app/classes/Transvision/Project.php b/app/classes/Transvision/Project.php
@@ -70,7 +70,8 @@ public static function getLastGaiaBranch()
     /**
      * Create a list of all supported repositories.
      *
-     * @return array list of supported repositories
+     * @return array List of supported repositories, key is the repo, value is
+     *               the nice name for the repo for display purposes.
      */
     public static function getSupportedRepositories()
     {
@@ -93,7 +94,7 @@ public static function getSupportedRepositories()
     /**
      * Get the list of repositories.
      *
-     * @return array list of local repositories
+     * @return array list of local repositories values
      */
     public static function getRepositories()
     {
@@ -168,6 +169,29 @@ public static function getRepositoryLocales($repository)
         return $supported_locales;
     }
 
+    /**
+     * Get the list of repositories available for a locale
+     *
+     * @param  string $locale Mozilla locale code
+     * @return array  A sorted list of repositories available for the locale
+     */
+    public static function getLocaleRepositories($locale)
+    {
+        $matches = [];
+        foreach (self::getRepositories() as $repository) {
+            if (in_array(
+                self::getLocaleInContext($locale, $repository),
+                self::getRepositoryLocales($repository)
+            )) {
+                $matches[] = $repository;
+            }
+        }
+
+        sort($matches);
+
+        return $matches;
+    }
+
     /**
      * Return the reference locale for a repository
      * We used to have en-GB as reference locale for mozilla.org
@@ -231,7 +255,10 @@ public static function getLocaleInContext($locale, $context)
         }
 
         // Firefox for iOS: no mapping
-        $locale_mappings['firefox_ios'] = [];
+        $locale_mappings['firefox_ios'] = [
+            'es-AR' => 'es',
+            'es-ES' => 'es',
+        ];
 
         // For other contexts use the same as Bugzilla
         $locale_mappings['other'] = $locale_mappings['bugzilla'];

diff --git a/app/classes/Transvision/ShowResults.php b/app/classes/Transvision/ShowResults.php
@@ -65,6 +65,9 @@ public static function getTranslationMemoryResults($entities, $array_strings, $s
             }
         }
 
+        // Remove duplicate results
+        $output = array_unique($output, SORT_REGULAR);
+
         // We sort by quality to get the best results first
         usort($output, function ($a, $b) {
            return $a['quality'] < $b['quality'];

diff --git a/app/models/api/repositories_list.php b/app/models/api/repositories_list.php
@@ -1,4 +1,10 @@
 <?php
 namespace Transvision;
 
+// We have a query for the repositories supported by a locale
+if (isset($request->parameters[2])) {
+    return $json = Project::getLocaleRepositories($request->parameters[2]);
+}
+
+// default to list all existing repositories
 return $json = Project::getRepositories();
diff --git a/app/models/api/repository_search.php b/app/models/api/repository_search.php
@@ -12,25 +12,26 @@
 
 // Get all strings
 $initial_search = urldecode(Utils::cleanString($request->parameters[6]));
-$source_strings = Utils::getRepoStrings($request->parameters[4], $request->parameters[3]);
-
-// Regex options
-$whole_word     = $get_option('whole_word') ? '\b' : '';
-$case_sensitive = $get_option('case_sensitive') ? '' : 'i';
-
-if ($get_option('perfect_match')) {
-    $regex = '~' . $whole_word . trim('^' . preg_quote($initial_search, '~') . '$') .
-             $whole_word . '~' . $case_sensitive . 'u';
-    if ($request->parameters[2] == 'entities') {
-        $entities = ShowResults::searchEntities($source_strings, $regex);
-        $source_strings = array_intersect_key($source_strings, array_flip($entities));
-    } else {
-        $source_strings = preg_grep($regex, $source_strings);
-        $entities = array_keys($source_strings);
-    }
-} else {
-    foreach (Utils::uniqueWords($initial_search) as $word) {
-        $regex = '~' . $whole_word . preg_quote($word, '~') .
+
+$repositories = $request->parameters[3] == 'global'
+    ? Project::getRepositories()
+    : [$request->parameters[3]];
+
+$entities_merged       = [];
+$source_strings_merged = [];
+$target_strings_merged = [];
+
+// We loop through all repositories searched and merge results
+foreach ($repositories as $repository) {
+    $source_strings = Utils::getRepoStrings($request->parameters[4], $repository);
+    // $source_strings = array_merge($source_strings, Utils::getRepoStrings($request->parameters[4], $repository));
+
+    // Regex options
+    $whole_word     = $get_option('whole_word') ? '\b' : '';
+    $case_sensitive = $get_option('case_sensitive') ? '' : 'i';
+
+    if ($get_option('perfect_match')) {
+        $regex = '~' . $whole_word . trim('^' . preg_quote($initial_search, '~') . '$') .
                  $whole_word . '~' . $case_sensitive . 'u';
         if ($request->parameters[2] == 'entities') {
             $entities = ShowResults::searchEntities($source_strings, $regex);
@@ -39,24 +40,40 @@
             $source_strings = preg_grep($regex, $source_strings);
             $entities = array_keys($source_strings);
         }
+    } else {
+        foreach (Utils::uniqueWords($initial_search) as $word) {
+            $regex = '~' . $whole_word . preg_quote($word, '~') .
+                     $whole_word . '~' . $case_sensitive . 'u';
+            if ($request->parameters[2] == 'entities') {
+                $entities = ShowResults::searchEntities($source_strings, $regex);
+                $source_strings = array_intersect_key($source_strings, array_flip($entities));
+            } else {
+                $source_strings = preg_grep($regex, $source_strings);
+                $entities = array_keys($source_strings);
+            }
+        }
     }
-}
 
-// We have our list of filtered source strings, get corresponding target locale strings
-$target_strings = array_intersect_key(
-    Utils::getRepoStrings($request->parameters[5], $request->parameters[3]),
-    array_flip($entities)
-);
+    // We have our list of filtered source strings, get corresponding target locale strings
+    $target_strings = array_intersect_key(
+        Utils::getRepoStrings($request->parameters[5], $repository),
+        array_flip($entities)
+    );
+
+    $source_strings_merged = array_merge($source_strings, $source_strings_merged);
+    $target_strings_merged = array_merge($target_strings, $target_strings_merged);
+    $entities_merged = array_merge($entities, $entities_merged);
+}
 
 // We sort arrays by key before array_splice() to keep matching keys
-ksort($source_strings);
-ksort($target_strings);
+ksort($source_strings_merged);
+ksort($target_strings_merged);
 
 // Limit results to 200
-array_splice($source_strings, 200);
-array_splice($target_strings, 200);
+array_splice($source_strings_merged, 500);
+array_splice($target_strings_merged, 500);
 
 return $json = ShowResults::getRepositorySearchResults(
-    $entities,
-    [$source_strings, $target_strings]
+    $entities_merged,
+    [$source_strings_merged, $target_strings_merged]
 );
diff --git a/app/models/api/translation_memory.php b/app/models/api/translation_memory.php
@@ -1,9 +1,12 @@
 <?php
 namespace Transvision;
 
-// get all strings
-$source_strings = Utils::getRepoStrings($request->parameters[3], $request->parameters[2]);
-$target_strings = Utils::getRepoStrings($request->parameters[4], $request->parameters[2]);
+$repositories = ($request->parameters[2] == 'global')
+    ? Project::getRepositories()
+    : [$request->parameters[2]];
+
+$source_strings_merged = [];
+$target_strings_merged = [];
 
 // The search
 $initial_search = Utils::cleanString($request->parameters[5]);
@@ -16,6 +19,21 @@
 $regex = $delimiter . $whole_word . $initial_search . $whole_word .
          $delimiter . $case_sensitive . 'u';
 
+// We loop through all repositories and merge the results
+foreach ($repositories as $repository) {
+    $source_strings = Utils::getRepoStrings($request->parameters[3], $repository);
+    $target_strings = Utils::getRepoStrings($request->parameters[4], $repository);
+
+    foreach ($terms as $word) {
+        $regex = $delimiter . $whole_word . preg_quote($word, $delimiter) .
+                 $whole_word . $delimiter . $case_sensitive . 'u';
+        $source_strings = preg_grep($regex, $source_strings);
+    }
+
+    $source_strings_merged = array_merge($source_strings, $source_strings_merged);
+    $target_strings_merged = array_merge($target_strings, $target_strings_merged);
+}
+
 // Closure to get extra parameters set
 $get_option = function ($option) use ($request) {
     $value = 0;
@@ -27,15 +45,9 @@
     return $value;
 };
 
-foreach ($terms as $word) {
-    $regex = $delimiter . $whole_word . preg_quote($word, $delimiter) .
-             $whole_word . $delimiter . $case_sensitive . 'u';
-    $source_strings = preg_grep($regex, $source_strings);
-}
-
 return $json = ShowResults::getTranslationMemoryResults(
-    array_keys($source_strings),
-    [$source_strings, $target_strings],
+    array_keys($source_strings_merged),
+    [$source_strings_merged, $target_strings_merged],
     $initial_search,
     $get_option('max_results'), // Cap results with the ?max_results=number option
     $get_option('min_quality') // Optional quality threshold defined by ?min_quality=50

diff --git a/tests/functional/api.php b/tests/functional/api.php
@@ -32,7 +32,10 @@
     ['v1/locales/central/', 200, '["ar","ast","cs","de","en-GB","en-US","eo","es-AR","es-CL","es-ES","es-MX","fa","fr","fy-NL","gl","he","hu","id","it","ja","ja-JP-mac","kk","ko","lt","lv","nb-NO","nl","nn-NO","pl","pt-BR","pt-PT","ru","sk","sl","sv-SE","th","tr","uk","vi","zh-CN","zh-TW"]'],
     ['v1/locales/iDontExist/', 400, '{"error":"The repo queried (iDontExist) doesn\'t exist."}'],
     ['v1/repositories/', 200, '["release","beta","aurora","central","firefox_ios","gaia_1_3","gaia_1_4","gaia_2_0","gaia_2_1","gaia_2_2","gaia","mozilla_org"]'],
+    ['v1/repositories/', 200, '["release","beta","aurora","central","firefox_ios","gaia_1_3","gaia_1_4","gaia_2_0","gaia_2_1","gaia_2_2","gaia","mozilla_org"]'],
+    ['v1/repositories/fr/', 200, '["aurora","beta","central","firefox_ios","gaia","gaia_1_3","gaia_1_4","gaia_2_0","gaia_2_1","gaia_2_2","mozilla_org","release"]'],
     ['v1/tm/central/en-US/fr/Bookmark/?max_results=3&min_quality=80', 200, '[{"source":"Bookmark","target":"Marquer cette page","quality":100},{"source":"Bookmark","target":"Marque-page","quality":100},{"source":"Bookmarks","target":"Marque-pages","quality":88.888888888889}]'],
+    ['v1/tm/global/fr/en-US/Ouvrir/', 200, '[{"source":"Ouvrir dans le Finder","target":"Find in Finder","quality":28.571428571429},{"source":"D\u00e9couvrez comment ouvrir une fen\u00eatre de navigation priv\u00e9e","target":"Learn how to open a private window","quality":8.7719298245614}]'],
     ['v1/versions/', 200, '{"v1":"stable"}'],
 ];
 

diff --git a/tests/testfiles/TMX/en-US/cache_en-US_mozilla_org.php b/tests/testfiles/TMX/en-US/cache_en-US_mozilla_org.php
@@ -0,0 +1,4 @@
+<?php
+$tmx = [
+'mozilla_org/firefox/android/index.lang:56a83c78' => 'Learn how to open a private window',
+];
diff --git a/tests/testfiles/TMX/fr/cache_fr_mozilla_org.php b/tests/testfiles/TMX/fr/cache_fr_mozilla_org.php
@@ -0,0 +1,4 @@
+<?php
+$tmx = [
+'mozilla_org/firefox/android/index.lang:56a83c78' => 'Découvrez comment ouvrir une fenêtre de navigation privée',
+];
diff --git a/tests/testfiles/config/mozilla_org.txt b/tests/testfiles/config/mozilla_org.txt
@@ -0,0 +1 @@
+fr
diff --git a/tests/units/Transvision/API.php b/tests/units/Transvision/API.php
@@ -109,6 +109,12 @@ public function isValidRequestDP()
             ['http://foobar/api/v1/tm/central/wrong_source/fr/hello world', false],
             ['http://foobar/api/v1/tm/central/en-US/wrong_target/hello world', false],
 
+            // repositories service
+            ['http://foobar/api/v1/repositories/', true],
+            ['http://foobar/api/v1/repositories/fr/', true],
+            ['http://foobar/api/v1/repositories/foobar/', false],
+            ['http://foobar/api/v1/repositories/en-US/', true],
+
             // versions service
             ['http://foobar/api/versions/', true],
         ];

diff --git a/tests/units/Transvision/Project.php b/tests/units/Transvision/Project.php
@@ -99,6 +99,25 @@ public function testGetRepositoryLocales($a, $b)
                 ->isEqualTo($b);
     }
 
+    public function getLocaleRepositoriesDP()
+    {
+        return [
+            ['fr', ['central', 'mozilla_org']],
+            ['foobar', []],
+        ];
+    }
+
+    /**
+     * @dataProvider getLocaleRepositoriesDP
+     */
+    public function testGetLocaleRepositories($a, $b)
+    {
+        $obj = new _Project();
+        $this
+            ->array($obj->getLocaleRepositories($a))
+                ->isEqualTo($b);
+    }
+
     public function testGetReferenceLocale()
     {
         $obj = new _Project();
@@ -141,6 +160,8 @@ public function getLocaleInContextDP()
             ['sr-Cyrl', 'mozilla_org', 'sr'],
             ['es-ES', 'foobar', 'es-ES'],
             ['fr', 'foobar', 'fr'],
+            ['es-ES', 'firefox_ios', 'es'],
+            ['es', 'firefox_ios', 'es'],
         ];
     }