From a2532527592e9fb21559ba3bf6e40a267e694df2 Mon Sep 17 00:00:00 2001 From: Pascal Chevrel Date: Wed, 21 Jan 2015 15:27:11 +0100 Subject: [PATCH] Issue #392: Translation memory API, query all repositories - add a new alias for all repos called 'global' - add Project::getRepositoriesLocale('ab-CD'), returns all the repositories available for a locale - new public api/v1/repositories// JSON API - 'global' search added to translation memory API + general search (strings and entities) API - Max results is now 500 for the API - dupes are removed from the translation memory API - update Project:getLocaleInContext() to have mappings for Spanishes - add new unit and functional tests --- app/classes/Transvision/API.php | 43 ++++++++-- app/classes/Transvision/Project.php | 33 +++++++- app/classes/Transvision/ShowResults.php | 3 + app/models/api/repositories_list.php | 6 ++ app/models/api/repository_search.php | 79 +++++++++++-------- app/models/api/translation_memory.php | 34 +++++--- tests/functional/api.php | 3 + .../TMX/en-US/cache_en-US_mozilla_org.php | 4 + .../testfiles/TMX/fr/cache_fr_mozilla_org.php | 4 + tests/testfiles/config/mozilla_org.txt | 1 + tests/units/Transvision/API.php | 6 ++ tests/units/Transvision/Project.php | 21 +++++ 12 files changed, 187 insertions(+), 50 deletions(-) create mode 100644 tests/testfiles/TMX/en-US/cache_en-US_mozilla_org.php create mode 100644 tests/testfiles/TMX/fr/cache_fr_mozilla_org.php create mode 100644 tests/testfiles/config/mozilla_org.txt diff --git a/app/classes/Transvision/API.php b/app/classes/Transvision/API.php index f9531bd4..80dcb2d6 100644 --- a/app/classes/Transvision/API.php +++ b/app/classes/Transvision/API.php @@ -45,7 +45,7 @@ class API public $error; public $logging = true; public $logger; - + public $valid_repositories; /** * The constructor analyzes the URL to extract its parameters * @@ -71,6 +71,8 @@ public function __construct($url) $this->extra_parameters = isset($url['query']) ? $this->getExtraParameters($url['query']) : []; + + $this->valid_repositories = array_merge(Project::getRepositories(), ['global']); } /** @@ -227,7 +229,7 @@ private function isValidServiceCall($service) return false; } - if (! $this->verifyRepositoryExists($this->parameters[3])) { + if (! $this->verifyRepositoryExists($this->parameters[3], true)) { return false; } @@ -261,7 +263,26 @@ private function isValidServiceCall($service) break; case 'repositories': // ex: api/repositories/ - // Generated from Project class, no user-defined variables = nothing to check + // ex: api/repositories/fr/ + // Generated from Project class + // There is one optional parameter, a locale code + if (isset($this->parameters[2])) { + $match = false; + + foreach (Project::getRepositories() as $repository) { + if ($this->verifyLocaleExists($this->parameters[2], $repository)) { + $match = true; + break; + } + } + + if (! $match) { + $this->log("The locale queried ({$this->parameters[2]}) is not supported"); + + return false; + } + } + break; case 'versions': // ex: api/versions/ @@ -302,11 +323,17 @@ private function verifyEnoughParameters($number) * Check that the repository asked for is one we support * * @param string $repository Name of the repository + * @param boolean $alias Do we allow aliases for repository names, + * ex: 'global', to query all repositories. Default to False * @return boolean True if we support this repository, False if we don't */ - private function verifyRepositoryExists($repository) + private function verifyRepositoryExists($repository, $alias = false) { - if (! in_array($repository, Project::getRepositories())) { + if (! in_array($repository, $this->valid_repositories)) { + $this->valid_repositories = $alias + ? array_merge(Project::getRepositories(), ['global']) + : Project::getRepositories(); + $this->log("The repo queried ({$repository}) doesn't exist."); return false; @@ -324,6 +351,12 @@ private function verifyRepositoryExists($repository) */ private function verifyLocaleExists($locale, $repository) { + if ($repository == 'global') { + if (! empty(Project::getLocaleRepositories($locale))) { + return true; + } + } + if (! in_array($locale, Project::getRepositoryLocales($repository))) { $this->log("The locale queried ({$locale}) is not " . "available for the repository ({$repository})."); diff --git a/app/classes/Transvision/Project.php b/app/classes/Transvision/Project.php index 2fc0afe5..5147ce80 100644 --- a/app/classes/Transvision/Project.php +++ b/app/classes/Transvision/Project.php @@ -70,7 +70,8 @@ public static function getLastGaiaBranch() /** * Create a list of all supported repositories. * - * @return array list of supported repositories + * @return array List of supported repositories, key is the repo, value is + * the nice name for the repo for display purposes. */ public static function getSupportedRepositories() { @@ -93,7 +94,7 @@ public static function getSupportedRepositories() /** * Get the list of repositories. * - * @return array list of local repositories + * @return array list of local repositories values */ public static function getRepositories() { @@ -168,6 +169,29 @@ public static function getRepositoryLocales($repository) return $supported_locales; } + /** + * Get the list of repositories available for a locale + * + * @param string $locale Mozilla locale code + * @return array A sorted list of repositories available for the locale + */ + public static function getLocaleRepositories($locale) + { + $matches = []; + foreach (self::getRepositories() as $repository) { + if (in_array( + self::getLocaleInContext($locale, $repository), + self::getRepositoryLocales($repository) + )) { + $matches[] = $repository; + } + } + + sort($matches); + + return $matches; + } + /** * Return the reference locale for a repository * We used to have en-GB as reference locale for mozilla.org @@ -231,7 +255,10 @@ public static function getLocaleInContext($locale, $context) } // Firefox for iOS: no mapping - $locale_mappings['firefox_ios'] = []; + $locale_mappings['firefox_ios'] = [ + 'es-AR' => 'es', + 'es-ES' => 'es', + ]; // For other contexts use the same as Bugzilla $locale_mappings['other'] = $locale_mappings['bugzilla']; diff --git a/app/classes/Transvision/ShowResults.php b/app/classes/Transvision/ShowResults.php index caf2da2c..d100c468 100644 --- a/app/classes/Transvision/ShowResults.php +++ b/app/classes/Transvision/ShowResults.php @@ -65,6 +65,9 @@ public static function getTranslationMemoryResults($entities, $array_strings, $s } } + // Remove duplicate results + $output = array_unique($output, SORT_REGULAR); + // We sort by quality to get the best results first usort($output, function ($a, $b) { return $a['quality'] < $b['quality']; diff --git a/app/models/api/repositories_list.php b/app/models/api/repositories_list.php index ef617884..c166121b 100644 --- a/app/models/api/repositories_list.php +++ b/app/models/api/repositories_list.php @@ -1,4 +1,10 @@ parameters[2])) { + return $json = Project::getLocaleRepositories($request->parameters[2]); +} + +// default to list all existing repositories return $json = Project::getRepositories(); diff --git a/app/models/api/repository_search.php b/app/models/api/repository_search.php index a58a6eca..e73ea5b2 100644 --- a/app/models/api/repository_search.php +++ b/app/models/api/repository_search.php @@ -12,25 +12,26 @@ // Get all strings $initial_search = urldecode(Utils::cleanString($request->parameters[6])); -$source_strings = Utils::getRepoStrings($request->parameters[4], $request->parameters[3]); - -// Regex options -$whole_word = $get_option('whole_word') ? '\b' : ''; -$case_sensitive = $get_option('case_sensitive') ? '' : 'i'; - -if ($get_option('perfect_match')) { - $regex = '~' . $whole_word . trim('^' . preg_quote($initial_search, '~') . '$') . - $whole_word . '~' . $case_sensitive . 'u'; - if ($request->parameters[2] == 'entities') { - $entities = ShowResults::searchEntities($source_strings, $regex); - $source_strings = array_intersect_key($source_strings, array_flip($entities)); - } else { - $source_strings = preg_grep($regex, $source_strings); - $entities = array_keys($source_strings); - } -} else { - foreach (Utils::uniqueWords($initial_search) as $word) { - $regex = '~' . $whole_word . preg_quote($word, '~') . + +$repositories = $request->parameters[3] == 'global' + ? Project::getRepositories() + : [$request->parameters[3]]; + +$entities_merged = []; +$source_strings_merged = []; +$target_strings_merged = []; + +// We loop through all repositories searched and merge results +foreach ($repositories as $repository) { + $source_strings = Utils::getRepoStrings($request->parameters[4], $repository); + // $source_strings = array_merge($source_strings, Utils::getRepoStrings($request->parameters[4], $repository)); + + // Regex options + $whole_word = $get_option('whole_word') ? '\b' : ''; + $case_sensitive = $get_option('case_sensitive') ? '' : 'i'; + + if ($get_option('perfect_match')) { + $regex = '~' . $whole_word . trim('^' . preg_quote($initial_search, '~') . '$') . $whole_word . '~' . $case_sensitive . 'u'; if ($request->parameters[2] == 'entities') { $entities = ShowResults::searchEntities($source_strings, $regex); @@ -39,24 +40,40 @@ $source_strings = preg_grep($regex, $source_strings); $entities = array_keys($source_strings); } + } else { + foreach (Utils::uniqueWords($initial_search) as $word) { + $regex = '~' . $whole_word . preg_quote($word, '~') . + $whole_word . '~' . $case_sensitive . 'u'; + if ($request->parameters[2] == 'entities') { + $entities = ShowResults::searchEntities($source_strings, $regex); + $source_strings = array_intersect_key($source_strings, array_flip($entities)); + } else { + $source_strings = preg_grep($regex, $source_strings); + $entities = array_keys($source_strings); + } + } } -} -// We have our list of filtered source strings, get corresponding target locale strings -$target_strings = array_intersect_key( - Utils::getRepoStrings($request->parameters[5], $request->parameters[3]), - array_flip($entities) -); + // We have our list of filtered source strings, get corresponding target locale strings + $target_strings = array_intersect_key( + Utils::getRepoStrings($request->parameters[5], $repository), + array_flip($entities) + ); + + $source_strings_merged = array_merge($source_strings, $source_strings_merged); + $target_strings_merged = array_merge($target_strings, $target_strings_merged); + $entities_merged = array_merge($entities, $entities_merged); +} // We sort arrays by key before array_splice() to keep matching keys -ksort($source_strings); -ksort($target_strings); +ksort($source_strings_merged); +ksort($target_strings_merged); // Limit results to 200 -array_splice($source_strings, 200); -array_splice($target_strings, 200); +array_splice($source_strings_merged, 500); +array_splice($target_strings_merged, 500); return $json = ShowResults::getRepositorySearchResults( - $entities, - [$source_strings, $target_strings] + $entities_merged, + [$source_strings_merged, $target_strings_merged] ); diff --git a/app/models/api/translation_memory.php b/app/models/api/translation_memory.php index c7d984a8..dcc178c5 100644 --- a/app/models/api/translation_memory.php +++ b/app/models/api/translation_memory.php @@ -1,9 +1,12 @@ parameters[3], $request->parameters[2]); -$target_strings = Utils::getRepoStrings($request->parameters[4], $request->parameters[2]); +$repositories = ($request->parameters[2] == 'global') + ? Project::getRepositories() + : [$request->parameters[2]]; + +$source_strings_merged = []; +$target_strings_merged = []; // The search $initial_search = Utils::cleanString($request->parameters[5]); @@ -16,6 +19,21 @@ $regex = $delimiter . $whole_word . $initial_search . $whole_word . $delimiter . $case_sensitive . 'u'; +// We loop through all repositories and merge the results +foreach ($repositories as $repository) { + $source_strings = Utils::getRepoStrings($request->parameters[3], $repository); + $target_strings = Utils::getRepoStrings($request->parameters[4], $repository); + + foreach ($terms as $word) { + $regex = $delimiter . $whole_word . preg_quote($word, $delimiter) . + $whole_word . $delimiter . $case_sensitive . 'u'; + $source_strings = preg_grep($regex, $source_strings); + } + + $source_strings_merged = array_merge($source_strings, $source_strings_merged); + $target_strings_merged = array_merge($target_strings, $target_strings_merged); +} + // Closure to get extra parameters set $get_option = function ($option) use ($request) { $value = 0; @@ -27,15 +45,9 @@ return $value; }; -foreach ($terms as $word) { - $regex = $delimiter . $whole_word . preg_quote($word, $delimiter) . - $whole_word . $delimiter . $case_sensitive . 'u'; - $source_strings = preg_grep($regex, $source_strings); -} - return $json = ShowResults::getTranslationMemoryResults( - array_keys($source_strings), - [$source_strings, $target_strings], + array_keys($source_strings_merged), + [$source_strings_merged, $target_strings_merged], $initial_search, $get_option('max_results'), // Cap results with the ?max_results=number option $get_option('min_quality') // Optional quality threshold defined by ?min_quality=50 diff --git a/tests/functional/api.php b/tests/functional/api.php index f52bbd35..1831df2b 100644 --- a/tests/functional/api.php +++ b/tests/functional/api.php @@ -32,7 +32,10 @@ ['v1/locales/central/', 200, '["ar","ast","cs","de","en-GB","en-US","eo","es-AR","es-CL","es-ES","es-MX","fa","fr","fy-NL","gl","he","hu","id","it","ja","ja-JP-mac","kk","ko","lt","lv","nb-NO","nl","nn-NO","pl","pt-BR","pt-PT","ru","sk","sl","sv-SE","th","tr","uk","vi","zh-CN","zh-TW"]'], ['v1/locales/iDontExist/', 400, '{"error":"The repo queried (iDontExist) doesn\'t exist."}'], ['v1/repositories/', 200, '["release","beta","aurora","central","firefox_ios","gaia_1_3","gaia_1_4","gaia_2_0","gaia_2_1","gaia_2_2","gaia","mozilla_org"]'], + ['v1/repositories/', 200, '["release","beta","aurora","central","firefox_ios","gaia_1_3","gaia_1_4","gaia_2_0","gaia_2_1","gaia_2_2","gaia","mozilla_org"]'], + ['v1/repositories/fr/', 200, '["aurora","beta","central","firefox_ios","gaia","gaia_1_3","gaia_1_4","gaia_2_0","gaia_2_1","gaia_2_2","mozilla_org","release"]'], ['v1/tm/central/en-US/fr/Bookmark/?max_results=3&min_quality=80', 200, '[{"source":"Bookmark","target":"Marquer cette page","quality":100},{"source":"Bookmark","target":"Marque-page","quality":100},{"source":"Bookmarks","target":"Marque-pages","quality":88.888888888889}]'], + ['v1/tm/global/fr/en-US/Ouvrir/', 200, '[{"source":"Ouvrir dans le Finder","target":"Find in Finder","quality":28.571428571429},{"source":"D\u00e9couvrez comment ouvrir une fen\u00eatre de navigation priv\u00e9e","target":"Learn how to open a private window","quality":8.7719298245614}]'], ['v1/versions/', 200, '{"v1":"stable"}'], ]; diff --git a/tests/testfiles/TMX/en-US/cache_en-US_mozilla_org.php b/tests/testfiles/TMX/en-US/cache_en-US_mozilla_org.php new file mode 100644 index 00000000..7c6840bf --- /dev/null +++ b/tests/testfiles/TMX/en-US/cache_en-US_mozilla_org.php @@ -0,0 +1,4 @@ + 'Learn how to open a private window', +]; diff --git a/tests/testfiles/TMX/fr/cache_fr_mozilla_org.php b/tests/testfiles/TMX/fr/cache_fr_mozilla_org.php new file mode 100644 index 00000000..c2545677 --- /dev/null +++ b/tests/testfiles/TMX/fr/cache_fr_mozilla_org.php @@ -0,0 +1,4 @@ + 'Découvrez comment ouvrir une fenêtre de navigation privée', +]; diff --git a/tests/testfiles/config/mozilla_org.txt b/tests/testfiles/config/mozilla_org.txt new file mode 100644 index 00000000..527e861b --- /dev/null +++ b/tests/testfiles/config/mozilla_org.txt @@ -0,0 +1 @@ +fr diff --git a/tests/units/Transvision/API.php b/tests/units/Transvision/API.php index 6e2f7091..028658f2 100644 --- a/tests/units/Transvision/API.php +++ b/tests/units/Transvision/API.php @@ -109,6 +109,12 @@ public function isValidRequestDP() ['http://foobar/api/v1/tm/central/wrong_source/fr/hello world', false], ['http://foobar/api/v1/tm/central/en-US/wrong_target/hello world', false], + // repositories service + ['http://foobar/api/v1/repositories/', true], + ['http://foobar/api/v1/repositories/fr/', true], + ['http://foobar/api/v1/repositories/foobar/', false], + ['http://foobar/api/v1/repositories/en-US/', true], + // versions service ['http://foobar/api/versions/', true], ]; diff --git a/tests/units/Transvision/Project.php b/tests/units/Transvision/Project.php index 368d76f4..a54fa959 100644 --- a/tests/units/Transvision/Project.php +++ b/tests/units/Transvision/Project.php @@ -99,6 +99,25 @@ public function testGetRepositoryLocales($a, $b) ->isEqualTo($b); } + public function getLocaleRepositoriesDP() + { + return [ + ['fr', ['central', 'mozilla_org']], + ['foobar', []], + ]; + } + + /** + * @dataProvider getLocaleRepositoriesDP + */ + public function testGetLocaleRepositories($a, $b) + { + $obj = new _Project(); + $this + ->array($obj->getLocaleRepositories($a)) + ->isEqualTo($b); + } + public function testGetReferenceLocale() { $obj = new _Project(); @@ -141,6 +160,8 @@ public function getLocaleInContextDP() ['sr-Cyrl', 'mozilla_org', 'sr'], ['es-ES', 'foobar', 'es-ES'], ['fr', 'foobar', 'fr'], + ['es-ES', 'firefox_ios', 'es'], + ['es', 'firefox_ios', 'es'], ]; }