Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Browse files
Add a view for commonly unlocalized words
- Loading branch information
1 parent
dfb232e
commit 2d70d2bfbc2fc44424b40e6f6e023f8ab57a3ac9
Showing
with
680 additions
and 0 deletions.
- +2 −0 README.md
- +20 −0 app/controllers/unlocalized_words.php
- +10 −0 app/inc/dispatcher.php
- +2 −0 app/inc/stop_word_list.php
- +2 −0 app/inc/urls.php
- +117 −0 app/models/unlocalized_words.php
- +1 −0 app/views/templates/base.php
- +30 −0 app/views/unlocalized_words.php
- +1 −0 tests/functional/pages.php
- +495 −0 web/js/sorttable.js
| @@ -0,0 +1,20 @@ | ||
| <?php | ||
| namespace Transvision; | ||
|
|
||
| // Get requested repo and locale. | ||
| require_once INC . 'l10n-init.php'; | ||
|
|
||
| // Include JS lib after $javascript_include gets reset in l10n-init.php. | ||
| $javascript_include = ['/js/sorttable.js']; | ||
|
|
||
| include MODELS . 'unlocalized_words.php'; | ||
|
|
||
| switch ($page) { | ||
| case 'unlocalized_json': | ||
| $json = $unlocalized_words; | ||
| include VIEWS . 'json.php'; | ||
| break; | ||
| default: | ||
| include VIEWS . 'unlocalized_words.php'; | ||
| break; | ||
| } |
| @@ -0,0 +1,2 @@ | ||
| <?php | ||
| $stopwords = ['a', 'about', 'above', 'above', 'across', 'after', 'afterwards', 'again', 'against', 'all', 'almost', 'alone', 'along', 'already', 'also', 'although', 'always', 'am', 'among', 'amongst', 'amoungst', 'amount', 'an', 'and', 'another', 'any', 'anyhow', 'anyone', 'anything', 'anyway', 'anywhere', 'are', 'around', 'as', 'at', 'back', 'be', 'became', 'because', 'become', 'becomes', 'becoming', 'been', 'before', 'beforehand', 'behind', 'being', 'below', 'beside', 'besides', 'between', 'beyond', 'bill', 'both', 'bottom', 'but', 'by', 'call', 'can', 'cannot', 'cant', 'co', 'con', 'could', 'couldnt', 'cry', 'de', 'describe', 'detail', 'do', 'done', 'down', 'due', 'during', 'each', 'eg', 'eight', 'either', 'eleven', 'else', 'elsewhere', 'empty', 'enough', 'etc', 'even', 'ever', 'every', 'everyone', 'everything', 'everywhere', 'except', 'few', 'fifteen', 'fify', 'fill', 'find', 'fire', 'first', 'five', 'for', 'former', 'formerly', 'forty', 'found', 'four', 'from', 'front', 'full', 'further', 'get', 'give', 'go', 'had', 'has', 'hasnt', 'have', 'he', 'hence', 'her', 'here', 'hereafter', 'hereby', 'herein', 'hereupon', 'hers', 'herself', 'him', 'himself', 'his', 'how', 'however', 'hundred', 'ie', 'if', 'in', 'inc', 'indeed', 'interest', 'into', 'is', 'it', 'its', 'itself', 'keep', 'last', 'latter', 'latterly', 'least', 'less', 'ltd', 'made', 'many', 'may', 'me', 'meanwhile', 'might', 'mill', 'mine', 'more', 'moreover', 'most', 'mostly', 'move', 'much', 'must', 'my', 'myself', 'name', 'namely', 'neither', 'never', 'nevertheless', 'next', 'nine', 'no', 'nobody', 'none', 'noone', 'nor', 'not', 'nothing', 'now', 'nowhere', 'of', 'off', 'often', 'on', 'once', 'one', 'only', 'onto', 'or', 'other', 'others', 'otherwise', 'our', 'ours', 'ourselves', 'out', 'over', 'own', 'part', 'per', 'perhaps', 'please', 'put', 'rather', 're', 'same', 'see', 'seem', 'seemed', 'seeming', 'seems', 'serious', 'several', 'she', 'should', 'show', 'side', 'since', 'sincere', 'six', 'sixty', 'so', 'some', 'somehow', 'someone', 'something', 'sometime', 'sometimes', 'somewhere', 'still', 'such', 'system', 'take', 'ten', 'than', 'that', 'the', 'their', 'them', 'themselves', 'then', 'thence', 'there', 'thereafter', 'thereby', 'therefore', 'therein', 'thereupon', 'these', 'they', 'thickv', 'thin', 'third', 'this', 'those', 'though', 'three', 'through', 'throughout', 'thru', 'thus', 'to', 'together', 'too', 'top', 'toward', 'towards', 'twelve', 'twenty', 'two', 'un', 'under', 'until', 'up', 'upon', 'us', 'very', 'via', 'was', 'we', 'well', 'were', 'what', 'whatever', 'when', 'whence', 'whenever', 'where', 'whereafter', 'whereas', 'whereby', 'wherein', 'whereupon', 'wherever', 'whether', 'which', 'while', 'whither', 'who', 'whoever', 'whole', 'whom', 'whose', 'why', 'will', 'with', 'within', 'without', 'would', 'yet', 'you', 'your', 'yours', 'yourself', 'yourselves', 'the']; |
| @@ -0,0 +1,117 @@ | ||
| <?php | ||
| namespace Transvision; | ||
|
|
||
| // Build arrays for the search form. | ||
| $channel_selector = Utils::getHtmlSelectOptions( | ||
| $repos_nice_names, | ||
| $repo, | ||
| true | ||
| ); | ||
| $target_locales_list = Utils::getHtmlSelectOptions( | ||
| Project::getRepositoryLocales($repo), | ||
| $locale | ||
| ); | ||
|
|
||
| $ref_locale = Project::getReferenceLocale($repo); | ||
| $unlocalized_words = []; | ||
| $skip_pspell = true; | ||
|
|
||
| /* | ||
| pspell helps getting rid of false positive results by keeping only valid | ||
| English words. The downside is that it’s filtering out 'jargon' words that | ||
| can be used in devtools or Mozilla-specific words. | ||
| */ | ||
| if (extension_loaded('pspell')) { | ||
| $pspell_link = \pspell_new('en_US', '', '', '', PSPELL_FAST); | ||
| $skip_pspell = false; | ||
| } else { | ||
| $logger->error('Please install libpspell-dev, php5-pspell and aspell-en ' . | ||
| 'packages and make sure pspell module is enabled in PHP config.'); | ||
| } | ||
|
|
||
| // Load reference strings. | ||
| $strings_reference = array_map('strtolower', Utils::getRepoStrings( | ||
| $ref_locale, | ||
| $repo | ||
| )); | ||
|
|
||
| // Load locale strings. | ||
| $strings_locale = array_map('strtolower', Utils::getRepoStrings($locale, $repo)); | ||
|
|
||
| /* | ||
| Go through all strings in $strings_reference, extract valid English words | ||
| then check if any of them is present in the localized string from | ||
| $strings_locale. | ||
| */ | ||
| foreach ($strings_reference as $string_ref_id => $ref_words) { | ||
| /* | ||
| If the string is missing in the locale or has been copy pasted from | ||
| source (e.g. not translated), skip it. | ||
| */ | ||
| if (! isset($strings_locale[$string_ref_id]) | ||
| || ($ref_words == $strings_locale[$string_ref_id] && $locale != $ref_locale)) { | ||
| continue; | ||
| } | ||
|
|
||
| $english_words = []; | ||
| $locale_words = $strings_locale[$string_ref_id]; | ||
|
|
||
| /* | ||
| Remove punctuation characters from the strings then explode them into | ||
| words. | ||
| */ | ||
| $ref_words = explode( | ||
| ' ', | ||
| preg_replace('/\p{P}/u', '', $ref_words) | ||
| ); | ||
| $locale_words = explode( | ||
| ' ', | ||
| preg_replace('/\p{P}/u', '', $locale_words) | ||
| ); | ||
|
|
||
| /* | ||
| Only keep valid English words with more than 1 character in the current | ||
| string. | ||
| */ | ||
| foreach ($ref_words as $word) { | ||
| if (strlen($word) > 1 && ! in_array($word, $english_words)) { | ||
| // Skip pspell when extension is not loaded | ||
| if ($skip_pspell) { | ||
| $english_words[] = $word; | ||
| continue; | ||
| } | ||
|
|
||
| if (pspell_check($pspell_link, $word)) { | ||
| $english_words[] = $word; | ||
| } | ||
| } | ||
| } | ||
|
|
||
| /* | ||
| Check if there is any English word in the current translated string and | ||
| count matches. | ||
| */ | ||
| foreach ($locale_words as $word) { | ||
| if (in_array($word, $english_words)) { | ||
| if (! isset($unlocalized_words[$word])) { | ||
| $unlocalized_words[$word] = 1; | ||
| } else { | ||
| $unlocalized_words[$word]++; | ||
| } | ||
| } | ||
| } | ||
| } | ||
| unset($strings_reference); | ||
| unset($strings_locale); | ||
|
|
||
| // Filtering out stop words from results at the end for performance reasons. | ||
| include INC . 'stop_word_list.php'; | ||
|
|
||
| foreach ($unlocalized_words as $key => $value) { | ||
| if (in_array($key, $stopwords)) { | ||
| unset($unlocalized_words[$key]); | ||
| } | ||
| } | ||
| unset($stopwords); | ||
|
|
||
| asort($unlocalized_words); |
| @@ -0,0 +1,30 @@ | ||
| <?php | ||
| namespace Transvision; | ||
|
|
||
| // Include the common simple search form | ||
| include __DIR__ . '/simplesearchform.php'; | ||
|
|
||
| $search_id = 'unlocalized_strings'; | ||
|
|
||
| $content = "<table class='collapsable results_table sortable {$search_id}'>\n" . | ||
| " <tr class='column_headers'>\n" . | ||
| " <th>English</th>\n" . | ||
| " <th>Occurrences</th>\n" . | ||
| " </tr>\n"; | ||
|
|
||
| foreach ($unlocalized_words as $string_id => $string_value) { | ||
| $link = "/?recherche={$string_id}&repo={$repo}&sourcelocale={$locale}" . | ||
| "&locale={$ref_locale}&search_type=strings&whole_word=whole_word"; | ||
|
|
||
| $link_title = $string_value == 1 | ||
| ? 'Search for this occurrence' | ||
| : 'Search for these occurrences'; | ||
|
|
||
| $content .= " <tr class='{$search_id}'>\n" . | ||
| " <td><a href='{$link}' title='{$link_title}'>{$string_id}</a></td>\n" . | ||
| " <td>{$string_value}</td>\n" . | ||
| " </tr>\n"; | ||
| } | ||
| $content .= "</table>\n"; | ||
|
|
||
| echo $content; |
Oops, something went wrong.