Skip to content

Commit 9e16a09

Browse files
flodoloflodolo
committed
Create view to display translation inconsistencies
1 parent 1ea9391 commit 9e16a09

File tree

14 files changed

+486
-115
lines changed

14 files changed

+486
-115
lines changed

‎app/classes/Transvision/Consistency.php‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
<?php
2+
namespace Transvision;
3+
4+
/**
5+
* Consistency class
6+
*
7+
* Functions used to analyze translation consistency
8+
*
9+
* @package Transvision
10+
*/
11+
class Consistency
12+
{
13+
/**
14+
* Find duplicated strings (strings with the same text, ignoring case).
15+
*
16+
* @param array $strings_array Array of strings in the form
17+
* string_id => string_value
18+
*
19+
* @return array Array of duplicated strings, same format as the
20+
* input array
21+
*/
22+
public static function findDuplicates($strings_array)
23+
{
24+
$duplicates = [];
25+
// Use array_filter to exclude empty strings
26+
$strings_array = array_filter($strings_array);
27+
asort($strings_array);
28+
29+
$previous_key = '';
30+
$previous_value = '';
31+
foreach ($strings_array as $key => $value) {
32+
if (strcasecmp($previous_value, $value) === 0) {
33+
$duplicates[$previous_key] = $previous_value;
34+
$duplicates[$key] = $value;
35+
}
36+
$previous_value = $value;
37+
$previous_key = $key;
38+
}
39+
40+
return $duplicates;
41+
}
42+
43+
/**
44+
* Filter out strings that should not be evaluted for consistency.
45+
*
46+
* @param array $strings_array Array of strings in the form
47+
* string_id => string_value
48+
* @param string Repository identifier
49+
*
50+
* @return array Array of filtered strings, with known false positives
51+
* removed
52+
*/
53+
public static function filterStrings($strings_array, $repo)
54+
{
55+
if (in_array($repo, Project::getDesktopRepositories())) {
56+
$repository_type = 'desktop';
57+
} elseif (in_array($repo, Project::getGaiaRepositories())) {
58+
$repository_type = 'gaia';
59+
} else {
60+
$repository_type = $repo;
61+
}
62+
63+
// Determine if a string should be excluded
64+
$ignore_string = function ($key, $value) use ($repository_type) {
65+
// Ignore strings of length 1 (e.g. accesskeys) and empty strings.
66+
if (strlen($value) <= 1) {
67+
return true;
68+
}
69+
70+
// Exclude CSS rules, "width:" or "height:"
71+
if (Strings::inString($value, ['width:', 'height:'])) {
72+
return true;
73+
}
74+
75+
// Exclude CSS width values like '38em'
76+
if (preg_match('/[\d|\.]+em/', $value)) {
77+
return true;
78+
}
79+
80+
if ($repository_type == 'gaia') {
81+
// Ignore plural forms containing [] in the key
82+
if (Strings::inString($key, ['[', ']'])) {
83+
return true;
84+
}
85+
86+
// Ignore accessibility strings
87+
if (strpos($key, 'accessibility.properties') !== false) {
88+
return true;
89+
}
90+
}
91+
92+
if ($repository_type == 'desktop') {
93+
/*
94+
Ignore some specific files:
95+
- AccessFu.properties: there are accessibility
96+
strings that remain identical for English in full and
97+
abbreviated form.
98+
- defines.inc (language pack attribution)
99+
- region.properties
100+
*/
101+
if (Strings::inString($key, ['AccessFu.properties', 'defines.inc', 'region.properties'])) {
102+
return true;
103+
}
104+
}
105+
106+
return false;
107+
};
108+
109+
foreach ($strings_array as $key => $value) {
110+
if ($ignore_string($key, $value)) {
111+
unset($strings_array[$key]);
112+
}
113+
}
114+
115+
return $strings_array;
116+
}
117+
}

‎app/classes/Transvision/Strings.php‎

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -64,15 +64,29 @@ public static function endsWith($haystack, $needles)
6464
}
6565

6666
/**
67-
* Check if $needle is in $haystack
67+
* Check if $needles are in $haystack
6868
*
69-
* @param string $haystack String to analyse
70-
* @param string $needle The string to look for
71-
* @return boolean True if the $haystack string contains $needle
69+
* @param string $haystack String to analyze
70+
* @param mixed $needles The string (or array of strings) to look for
71+
* @param boolean $match_all True if we need to match all $needles, false
72+
* if it's enough to match one. Default: false
73+
*
74+
* @return boolean True if the $haystack string contains any/all $needles
7275
*/
73-
public static function inString($haystack, $needle)
76+
public static function inString($haystack, $needles, $match_all = false)
7477
{
75-
return mb_strpos($haystack, $needle, $offset = 0, 'UTF-8') !== false ? true : false;
78+
$matches = 0;
79+
foreach ((array) $needles as $needle) {
80+
if (mb_strpos($haystack, $needle, $offset = 0, 'UTF-8') !== false) {
81+
// If I need to match any needle, I can stop at the first match
82+
if (! $match_all) {
83+
return true;
84+
}
85+
$matches++;
86+
}
87+
}
88+
89+
return $matches == count($needles);
7690
}
7791

7892
/**

‎app/controllers/consistency.php‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
<?php
2+
namespace Transvision;
3+
4+
// Get requested repo and locale
5+
require_once INC . 'l10n-init.php';
6+
7+
include MODELS . 'consistency.php';
8+
include VIEWS . 'consistency.php';

‎app/inc/dispatcher.php‎

Lines changed: 42 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -20,38 +20,27 @@
2020
$page_title = '3 locales search';
2121
$page_descr = 'One source locale, get search results for two target locales';
2222
break;
23-
case 'news':
24-
$controller = 'changelog';
25-
$page_title = 'Transvision News and Release Notes';
26-
$page_descr = '';
27-
$css_include = ['changelog.css'];
23+
case 'accesskeys':
24+
$view = 'accesskeys';
25+
$page_title = 'Access Keys';
26+
$page_descr = 'Check your access keys.';
2827
break;
29-
case 'rss':
30-
$controller = 'changelog';
28+
case Strings::StartsWith($url['path'], 'api'):
29+
$controller = 'api';
30+
$page_title = 'API response';
31+
$page_descr = '';
3132
$template = false;
3233
break;
33-
case 'stats':
34-
$view = 'showrepos';
35-
$page_title = 'Status Overview';
36-
$page_descr = 'Repository status overview.';
37-
break;
38-
case 'repocomparison':
39-
$view = 'repocomparison';
40-
break;
41-
case 'gaia':
42-
$view = 'gaia';
43-
$page_title = 'Gaia Comparison';
44-
$page_descr = 'Check the Status of your GAIA strings across repositories.';
45-
break;
4634
case 'channelcomparison':
4735
$controller = 'channelcomparison';
4836
$page_title = 'Channel Comparison';
4937
$page_descr = 'Compare strings from channel to channel.';
5038
break;
51-
case 'accesskeys':
52-
$view = 'accesskeys';
53-
$page_title = 'Access Keys';
54-
$page_descr = 'Check your access keys.';
39+
case 'consistency':
40+
$experimental = true;
41+
$controller = 'consistency';
42+
$page_title = 'Translation Consistency';
43+
$page_descr = 'Analyze translation consistency across repositories.';
5544
break;
5645
case 'credits':
5746
$view = 'credits';
@@ -64,13 +53,42 @@
6453
$page_descr = 'Create and download your own <abbr title="Translation Memory eXchange">TMX</abbr> file containing the strings you need.';
6554
$css_include = ['tmx.css'];
6655
break;
56+
case 'gaia':
57+
$view = 'gaia';
58+
$page_title = 'Gaia Comparison';
59+
$page_descr = 'Check the Status of your GAIA strings across repositories.';
60+
break;
61+
case 'news':
62+
$controller = 'changelog';
63+
$page_title = 'Transvision News and Release Notes';
64+
$page_descr = '';
65+
$css_include = ['changelog.css'];
66+
break;
67+
case 'productization':
68+
$view = 'productization';
69+
$page_title = 'Productization Overview';
70+
$page_descr = 'Show productization aspects for this locale.';
71+
$css_include = ['productization.css'];
72+
break;
73+
case 'repocomparison':
74+
$view = 'repocomparison';
75+
break;
76+
case 'rss':
77+
$controller = 'changelog';
78+
$template = false;
79+
break;
6780
case 'showrepos':
6881
$experimental = true;
6982
$controller = 'health_status';
7083
$page_title = 'Health status';
7184
$page_descr = 'Check the health status of locales.';
7285
$css_include = ['health.css'];
7386
break;
87+
case 'stats':
88+
$view = 'showrepos';
89+
$page_title = 'Status Overview';
90+
$page_descr = 'Repository status overview.';
91+
break;
7492
case 'string':
7593
$controller = 'onestring';
7694
$page_title = 'All translations for this string:';
@@ -86,18 +104,6 @@
86104
$page_title = 'Variables Overview';
87105
$page_descr = 'Show potential errors related to missing or mispelled variables in your strings.';
88106
break;
89-
case 'productization':
90-
$view = 'productization';
91-
$page_title = 'Productization Overview';
92-
$page_descr = 'Show productization aspects for this locale.';
93-
$css_include = ['productization.css'];
94-
break;
95-
case Strings::StartsWith($url['path'], 'api'):
96-
$controller = 'api';
97-
$page_title = 'API response';
98-
$page_descr = '';
99-
$template = false;
100-
break;
101107
default:
102108
$controller = 'mainsearch';
103109
break;

‎app/inc/urls.php‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
'rss' => 'rss',
77
'stats' => 'stats',
88
'channelcomparison' => 'channelcomp',
9+
'consistency' => 'consistency',
910
'accesskeys' => 'keys',
1011
'credits' => 'credits',
1112
'downloads' => 'downloads',

‎app/models/consistency.php‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
<?php
2+
namespace Transvision;
3+
4+
// Build arrays for the search form, ignore mozilla.org
5+
$channel_selector = Utils::getHtmlSelectOptions(
6+
array_diff($repos_nice_names, ['mozilla.org']),
7+
$repo,
8+
true
9+
);
10+
11+
$target_locales_list = Utils::getHtmlSelectOptions(
12+
Project::getRepositoryLocales($repo),
13+
$locale
14+
);
15+
16+
$reference_locale = Project::getReferenceLocale($repo);
17+
18+
// Set a default for the number of strings to display
19+
$strings_number = 0;
20+
21+
$source_strings = Utils::getRepoStrings($reference_locale, $repo);
22+
// Ignore empty strings in translations
23+
$target_strings = array_filter(Utils::getRepoStrings($locale, $repo));
24+
25+
if (! empty($source_strings)) {
26+
// Remove known problematic strings
27+
$duplicated_strings_english = Consistency::filterStrings($source_strings, $repo);
28+
// Find strings that are identical in English
29+
$duplicated_strings_english = Consistency::findDuplicates($duplicated_strings_english);
30+
}
31+
32+
if (! empty($duplicated_strings_english)) {
33+
$inconsistent_translations = [];
34+
$available_translations = [];
35+
36+
/*
37+
Using CachingIterator to be able to look ahead during the foreach
38+
cycle, since it's an associative array.
39+
http://php.net/manual/en/class.cachingiterator.php
40+
*/
41+
$collection = new \CachingIterator(new \ArrayIterator($duplicated_strings_english));
42+
foreach ($collection as $key => $value) {
43+
// Ignore this string if is not available in the localization
44+
if (! isset($target_strings[$key])) {
45+
$available_translations = [];
46+
continue;
47+
}
48+
49+
$available_translations[] = $target_strings[$key];
50+
/*
51+
If the current English string is different from the previous one,
52+
or I am at the last element, I need to store it with all available
53+
translations collected so far.
54+
55+
$collection->getInnerIterator()->current() stores the value of
56+
the next element.
57+
*/
58+
if (! $collection->hasNext() || $collection->getInnerIterator()->current() != $value) {
59+
$available_translations = array_unique($available_translations);
60+
/*
61+
Store this string only if there's more than one translation.
62+
If there's only one, translations are consistent.
63+
*/
64+
if (count($available_translations) > 1) {
65+
$inconsistent_translations[] = [
66+
'source' => $value,
67+
'target' => $available_translations,
68+
];
69+
}
70+
$available_translations = [];
71+
}
72+
}
73+
74+
$strings_number = count($inconsistent_translations);
75+
}

‎app/scripts/generate_sources‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -59,12 +59,10 @@ if ($gaia_versions) {
5959
// Create a JSON file with the list of all supported repositories in the correct order
6060
$json_repositories = [];
6161
foreach ($repositories as $repository) {
62-
if ($repository['type'] != 'test') {
63-
$json_repositories[intval($repository['display_order'])] = [
64-
'id' => $repository['id'],
65-
'name' => $repository['name']
66-
];
67-
}
62+
$json_repositories[intval($repository['display_order'])] = [
63+
'id' => $repository['id'],
64+
'name' => $repository['name']
65+
];
6866
}
6967
ksort($json_repositories);
7068
echo "* Saving JSON record of all supported repositories\n";

0 commit comments

Comments
 (0)