Skip to content

Commit 91609fa

Browse files
flodoloflodolo
committed
Restructure TMX folders
Move from /TMX/repo/locale to /TMX/locale setup.sh * Check existence of /TMX/locale * Switch URLs to https tmxmaker.py * Use meaningful variable names * Use argparse instead of OptionParser (deprecated since 2.7) * Store cache files in TMX/locale, don't create duplicated en-US cache files mozorg.php * Move script from /app/inc/mozorg.php to /app/scripts/tmx_mozillaorg * Generate cache file in /TMX/locale/cache_locale_mozilla_org.php Utils::getRepoStrings * Update cache file path and tests Dotlang::getStrings * Manage reference locale differently (store original string instead of empty)
1 parent c79f342 commit 91609fa

File tree

12 files changed

+101
-93
lines changed

12 files changed

+101
-93
lines changed

‎app/classes/Transvision/Dotlang.php‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ public static function getLangFilesList($path)
4646
}
4747

4848
/**
49-
* Loads the lang file and returns a cleaned up array of the lines
49+
* Loads the lang file and returns a cleaned up array of the lines
5050
*
5151
* @param string $file path to the local file to load
5252
* @return array All the significant lines in the file as an array
@@ -73,10 +73,11 @@ public static function getFile($file)
7373
* ;String in english
7474
* translated string
7575
*
76-
* @param string $file .lang file to load
77-
* @return array Array of strings as [original => translation]
76+
* @param string $file .lang file to load
77+
* @param boolean $reference_locale True if the current locale is the reference locale
78+
* @return array Array of strings as [original => translation]
7879
*/
79-
public static function getStrings($file)
80+
public static function getStrings($file, $reference_locale)
8081
{
8182
// get all the lines in an array
8283
$f = self::getFile($file);
@@ -104,7 +105,12 @@ public static function getStrings($file)
104105
}
105106

106107
// If untranslated, I need to store an empty string as translation
107-
$strings[$english] = ($translation == $english) ? '' : $translation;
108+
// unless it's the reference locale
109+
if ($reference_locale) {
110+
$strings[$english] = $english;
111+
} else {
112+
$strings[$english] = ($translation == $english) ? '' : $translation;
113+
}
108114

109115
$i++;
110116
}

‎app/classes/Transvision/Utils.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -230,7 +230,7 @@ public static function getRepoStrings($locale, $repository)
230230
{
231231
$locale = Project::getLocaleInContext($locale, $repository);
232232

233-
$file = TMX . "{$repository}/{$locale}/cache_{$locale}.php";
233+
$file = TMX . "{$locale}/cache_{$locale}_{$repository}.php";
234234

235235
if (! $tmx = Cache::getKey($file)) {
236236
if (is_file($file)) {

‎app/scripts/glossaire.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -278,7 +278,7 @@ function updateFromSVN() {
278278
then
279279
echogreen "Extract strings on svn"
280280
cd $install
281-
nice -20 php app/inc/mozorg.php
281+
nice -20 app/scripts/tmx_mozillaorg
282282
fi
283283
}
284284

@@ -296,7 +296,7 @@ do
296296
updateGaiaRepo ${gaia_version}
297297
done
298298

299-
# mozilla.org has its own extraction script, mozorg.php
299+
# mozilla.org has its own extraction script
300300
updateFromSVN
301301

302302
# Generate productization data

‎app/scripts/setup.sh‎

Lines changed: 24 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ eval $(cat $DIR/../config/config.ini | $DIR/ini_to_bash.py)
3030

3131
# Generate sources (gaia versions, supported locales)
3232
echogreen "Generate list of locales and supported Gaia versions"
33-
php $DIR/generate_sources $config
33+
$DIR/generate_sources $config
3434

3535
# Check if we have sources
3636
echogreen "Checking if Transvision sources are available..."
@@ -117,7 +117,7 @@ function checkoutSilme() {
117117
then
118118
echogreen "Checking out the SILME library into $libraries"
119119
cd $libraries
120-
hg clone http://hg.mozilla.org/l10n/silme -u silme-0.8.0
120+
hg clone https://hg.mozilla.org/l10n/silme -u silme-0.8.0
121121
cd $install
122122
fi
123123
}
@@ -127,11 +127,11 @@ function initDesktopL10nRepo() {
127127
if [ "$1" == "central" ]
128128
then
129129
local repo_folder="trunk_l10n"
130-
local repo_path="http://hg.mozilla.org/l10n-central"
130+
local repo_path="https://hg.mozilla.org/l10n-central"
131131
local locale_list="trunk_locales"
132132
else
133133
local repo_folder="${1}_l10n"
134-
local repo_path="http://hg.mozilla.org/releases/l10n/mozilla-$1"
134+
local repo_path="https://hg.mozilla.org/releases/l10n/mozilla-$1"
135135
local locale_list="${1}_locales"
136136
fi
137137

@@ -152,12 +152,11 @@ function initDesktopL10nRepo() {
152152
hg clone $repo_path/$locale $locale
153153
fi
154154

155-
if [ ! -d $root/TMX/$1/$locale ]
155+
if [ ! -d $root/TMX/$locale ]
156156
then
157157
# ${1^^} = uppercase($1)
158-
echogreen "Creating this locale cache for ${1^^}:"
159-
echogreen $locale
160-
mkdir -p $root/TMX/$1/$locale
158+
echogreen "Creating folder cache for: $locale"
159+
mkdir -p $root/TMX/$locale
161160
fi
162161
done
163162
}
@@ -170,20 +169,20 @@ function initDesktopSourceRepo() {
170169
echogreen "Checking out the following repo:"
171170
echogreen $trunk_source/comm-central/
172171
cd $trunk_source
173-
hg clone http://hg.mozilla.org/comm-central/
172+
hg clone https://hg.mozilla.org/comm-central/
174173
fi
175174

176175
if [ ! -d $trunk_source/mozilla-central/.hg ]
177176
then
178177
echogreen "Checking out the following repo:"
179178
echogreen $trunk_source/mozilla-central/
180179
cd $trunk_source
181-
hg clone http://hg.mozilla.org/mozilla-central/
180+
hg clone https://hg.mozilla.org/mozilla-central/
182181
fi
183182

184183
# Checkout chatzilla and venkman only on trunk, since they don't
185184
# have branches. Can add other products to array nobranch_products,
186-
# as long as their code is located in http://hg.mozilla.org/PRODUCT
185+
# as long as their code is located in https://hg.mozilla.org/PRODUCT
187186
local nobranch_products=( chatzilla venkman )
188187
for product in "${nobranch_products[@]}"
189188
do
@@ -192,7 +191,7 @@ function initDesktopSourceRepo() {
192191
echogreen "Checking out the following repo:"
193192
echogreen $trunk_source/$product
194193
cd $trunk_source
195-
hg clone http://hg.mozilla.org/$product/
194+
hg clone https://hg.mozilla.org/$product/
196195
fi
197196
done
198197
else
@@ -203,22 +202,21 @@ function initDesktopSourceRepo() {
203202
then
204203
echogreen "Checking out the following repo:"
205204
echogreen ${!target}/comm-$1/
206-
hg clone http://hg.mozilla.org/releases/comm-$1/
205+
hg clone https://hg.mozilla.org/releases/comm-$1/
207206
fi
208207

209208
if [ ! -d ${!target}/mozilla-$1/.hg ]
210209
then
211210
echogreen "Checking out the following repo:"
212211
echogreen ${!target}/mozilla-$1/
213-
hg clone http://hg.mozilla.org/releases/mozilla-$1/
212+
hg clone https://hg.mozilla.org/releases/mozilla-$1/
214213
fi
215214
fi
216215

217-
if [ ! -d $root/TMX/$1/en-US ]
216+
if [ ! -d $root/TMX/en-US ]
218217
then
219-
echogreen "Creating this locale cache for $1:"
220-
echogreen en-US
221-
mkdir -p $root/TMX/$1/en-US
218+
echogreen "Creating folder cache for: en-US"
219+
mkdir -p $root/TMX/en-US
222220
fi
223221
}
224222

@@ -229,13 +227,13 @@ function initGaiaRepo () {
229227
local locale_list="gaia_locales"
230228
local repo_name="gaia"
231229
local repo_pretty_name="Gaia"
232-
local repo_path="http://hg.mozilla.org/gaia-l10n"
230+
local repo_path="https://hg.mozilla.org/gaia-l10n"
233231
else
234232
local locale_list="gaia_locales_$1"
235233
local repo_name="gaia_$1"
236234
# If version is "1_4", repo_pretty_name will be "Gaia 1.4"
237235
local repo_pretty_name="Gaia ${1/_/.}"
238-
local repo_path="http://hg.mozilla.org/releases/gaia-l10n/v$1"
236+
local repo_path="https://hg.mozilla.org/releases/gaia-l10n/v$1"
239237
fi
240238

241239
echogreen "$repo_pretty_name initialization"
@@ -255,11 +253,10 @@ function initGaiaRepo () {
255253
hg clone $repo_path/$locale
256254
fi
257255

258-
if [ ! -d $root/TMX/$repo_name/$locale ]
256+
if [ ! -d $root/TMX/$locale ]
259257
then
260-
echogreen "Creating this locale cache for $repo_pretty_name:"
261-
echogreen $locale
262-
mkdir -p $root/TMX/$repo_name/$locale
258+
echogreen "Creating folder cache for: $locale"
259+
mkdir -p $root/TMX/$locale
263260
fi
264261
done
265262
fi
@@ -309,11 +306,10 @@ fi
309306

310307
for locale in $(cat $l20n_test_locales)
311308
do
312-
if [ ! -d $root/TMX/l20n_test/$locale ]
309+
if [ ! -d $root/TMX/$locale ]
313310
then
314-
echogreen "Creating this locale cache for L20n test:"
315-
echogreen $locale
316-
mkdir -p $root/TMX/l20n_test/$locale
311+
echogreen "Creating locale cache for: $locale"
312+
mkdir -p $root/TMX/$locale
317313
fi
318314
done
319315

app/inc/mozorg.php renamed to app/scripts/tmx_mozillaorg

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#!/usr/bin/env php
12
<?php
23
namespace Transvision;
34

@@ -6,7 +7,7 @@
67
die('Nope');
78
}
89

9-
include __DIR__ . '/init.php';
10+
include __DIR__ . '/../inc/init.php';
1011

1112
error_log('Mozilla.org: extraction of strings');
1213

@@ -20,11 +21,14 @@ function ($item) use ($path) {
2021
},
2122
$mozilla_org_files);
2223

23-
$out_english = $out_translation = '';
24+
$out_translation = '';
2425
$total_strings = 0;
2526

27+
// Check if this is the reference locale
28+
$reference_locale = (Project::getReferenceLocale('mozilla_org') == $locale);
29+
2630
foreach ($mozilla_org_files as $file) {
27-
$strings = Dotlang::getStrings(SVN . "mozilla_org/{$locale}/{$file}");
31+
$strings = Dotlang::getStrings(SVN . "mozilla_org/{$locale}/{$file}", $reference_locale);
2832
$total_strings += count($strings);
2933

3034
foreach ($strings as $english => $translation) {
@@ -40,15 +44,12 @@ function ($item) use ($path) {
4044
return $array_line;
4145
};
4246

43-
$out_english .= $output($english, $english);
4447
$out_translation .= $output($english, $translation);
4548
}
4649
}
4750

48-
$out_english = "<?php\n\$tmx = [\n" . $out_english . "];\n";
4951
$out_translation = "<?php\n\$tmx = [\n" . $out_translation . "];\n";
5052

51-
Files::fileForceContents(TMX . "mozilla_org/{$locale}/cache_{$locale}.php", $out_translation);
52-
Files::fileForceContents(TMX . "mozilla_org/{$locale}/cache_en-US.php", $out_english);
53+
Files::fileForceContents(TMX . "{$locale}/cache_{$locale}_mozilla_org.php", $out_translation);
5354
error_log("{$locale}: {$total_strings} strings");
5455
}

‎app/scripts/tmxmaker.py‎

Lines changed: 36 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
#!/usr/bin/python
22

3+
import argparse
4+
import datetime
35
import os
46
import sys
5-
import datetime
6-
from optparse import OptionParser
77
from ConfigParser import SafeConfigParser
88

99
""" Here we read the server configuration file """
@@ -57,71 +57,65 @@ def php_close_array(target_file):
5757
target_file.write("];\n")
5858

5959
if __name__ == "__main__":
60-
usage = "test"
61-
parser = OptionParser(usage, version='%prog 0.1')
62-
(options, args) = parser.parse_args(sys.argv[1:])
63-
64-
locale_repo = args[0]
65-
en_US_repo = args[1]
66-
langcode1 = args[2]
67-
langcode2 = args[3]
68-
repo = args[4]
60+
# Read command line input parameters
61+
parser = argparse.ArgumentParser()
62+
parser.add_argument('locale_repo', help='Path to locale files')
63+
parser.add_argument('reference_repo', help='Path to reference files')
64+
parser.add_argument('locale_code', help='Locale language code')
65+
parser.add_argument('reference_code', help='Reference language code')
66+
parser.add_argument('repository', help='Repository name')
67+
args = parser.parse_args()
6968

7069
exclusionlist = ['.hgtags', '.hg', '.git', '.gitignore']
71-
dirs1 = os.listdir(locale_repo)
72-
if repo.startswith('gaia') or repo == 'l20n_test' :
73-
dirs2 = os.listdir(en_US_repo)
74-
dirs2 = [x for x in dirs2 if x not in exclusionlist]
70+
dirs_locale = os.listdir(args.locale_repo)
71+
if args.repository.startswith('gaia') or args.repository == 'l20n_test' :
72+
dirs_reference = os.listdir(args.reference_repo)
73+
dirs_reference = [x for x in dirs_reference if x not in exclusionlist]
7574
else:
76-
dirs2 = ["browser", "calendar", "chat", "dom", "editor",
77-
"extensions", "mail", "mobile", "netwerk", "other-licenses",
78-
"security", "services", "suite", "toolkit", "webapprt"]
75+
dirs_reference = [
76+
"browser", "calendar", "chat", "dom", "editor",
77+
"extensions", "mail", "mobile", "netwerk", "other-licenses",
78+
"security", "services", "suite", "toolkit", "webapprt"
79+
]
7980

80-
dirs = filter(lambda x:x in dirs1, dirs2)
81+
dirs = filter(lambda x:x in dirs_locale, dirs_reference)
8182

82-
localpath = os.path.join(localdir, repo, langcode1)
83-
filename1 = os.path.join(localpath, "cache_" + langcode2 + ".php")
84-
filename2 = os.path.join(localpath, "cache_" + langcode1 + ".php")
83+
localpath = os.path.join(localdir, args.locale_code)
84+
filename_locale = os.path.join(localpath, "cache_%s_%s.php" % (args.locale_code, args.repository))
8585

86-
target_file1 = open(filename1, "w")
87-
target_file2 = open(filename2, "w")
88-
php_header(target_file1)
89-
php_header(target_file2)
86+
target_locale = open(filename_locale, "w")
87+
php_header(target_locale)
9088

9189
for directory in dirs:
92-
93-
path1 = en_US_repo + directory
94-
path2 = locale_repo + directory
90+
path_reference = args.reference_repo + directory
91+
path_locale = args.locale_repo + directory
9592

9693
rcsClient = silme.io.Manager.get('file')
9794
try:
98-
l10nPackage = rcsClient.get_package(path1, object_type='entitylist')
95+
l10nPackage_reference = rcsClient.get_package(path_reference, object_type='entitylist')
9996
except:
100-
print 'Silme couldn\'t extract data for ' + path1
97+
print 'Silme couldn\'t extract data for ' + path_reference
10198
continue
10299

103100
try:
104-
l10nPackage2 = rcsClient.get_package(path2, object_type='entitylist')
101+
l10nPackage_locale = rcsClient.get_package(path_locale, object_type='entitylist')
105102
except:
106-
print 'Silme couldn\'t extract data for ' + path2
103+
print 'Silme couldn\'t extract data for ' + path_locale
107104
continue
108105

109106
strings = {}
110-
chaine = get_string(l10nPackage, directory)
107+
strings_reference = get_string(l10nPackage_reference, directory)
111108

112109
"""
113110
get_string() is a recursive function that fills 'strings', a global array
114111
We need to reset that global array before calling the function again
115112
"""
116113
del strings
117114
strings = {}
118-
chaine2 = get_string(l10nPackage2, directory)
115+
strings_locale = get_string(l10nPackage_locale, directory)
119116

117+
for entity in strings_reference:
118+
php_add_to_array(entity, strings_locale.get(entity, ""), target_locale)
120119

121-
for entity in chaine:
122-
php_add_to_array(entity, chaine[entity], target_file1)
123-
php_add_to_array(entity, chaine2.get(entity, ""), target_file2)
124-
php_close_array(target_file1)
125-
php_close_array(target_file2)
126-
target_file1.close()
127-
target_file2.close()
120+
php_close_array(target_locale)
121+
target_locale.close()

tests/testfiles/TMX/central/en-US/cache_en-US.php renamed to tests/testfiles/TMX/en-US/cache_en-US_central.php

File renamed without changes.

tests/testfiles/TMX/gaia/es/cache_es.php renamed to tests/testfiles/TMX/es/cache_es_gaia.php

File renamed without changes.

tests/testfiles/TMX/central/fr/cache_fr.php renamed to tests/testfiles/TMX/fr/cache_fr_central.php

File renamed without changes.

0 commit comments

Comments
 (0)