Permalink
Browse files

Restructure TMX folders

Move from /TMX/repo/locale to /TMX/locale

setup.sh
* Check existence of /TMX/locale
* Switch URLs to https

tmxmaker.py
* Use meaningful variable names
* Use argparse instead of OptionParser (deprecated since 2.7)
* Store cache files in TMX/locale, don't create duplicated en-US cache files

mozorg.php
* Move script from /app/inc/mozorg.php to /app/scripts/tmx_mozillaorg
* Generate cache file in /TMX/locale/cache_locale_mozilla_org.php

Utils::getRepoStrings
* Update cache file path and tests

Dotlang::getStrings
* Manage reference locale differently (store original string instead of empty)
  • Loading branch information...
flodolo committed Jun 30, 2015
1 parent c79f342 commit 91609fa8c0b72f1c1317f866992087de134135b5
@@ -46,7 +46,7 @@ public static function getLangFilesList($path)
}
/**
- * Loads the lang file and returns a cleaned up array of the lines
+ * Loads the lang file and returns a cleaned up array of the lines
*
* @param string $file path to the local file to load
* @return array All the significant lines in the file as an array
@@ -73,10 +73,11 @@ public static function getFile($file)
* ;String in english
* translated string
*
- * @param string $file .lang file to load
- * @return array Array of strings as [original => translation]
+ * @param string $file .lang file to load
+ * @param boolean $reference_locale True if the current locale is the reference locale
+ * @return array Array of strings as [original => translation]
*/
- public static function getStrings($file)
+ public static function getStrings($file, $reference_locale)
{
// get all the lines in an array
$f = self::getFile($file);
@@ -104,7 +105,12 @@ public static function getStrings($file)
}
// If untranslated, I need to store an empty string as translation
- $strings[$english] = ($translation == $english) ? '' : $translation;
+ // unless it's the reference locale
+ if ($reference_locale) {
+ $strings[$english] = $english;
+ } else {
+ $strings[$english] = ($translation == $english) ? '' : $translation;
+ }
$i++;
}
@@ -230,7 +230,7 @@ public static function getRepoStrings($locale, $repository)
{
$locale = Project::getLocaleInContext($locale, $repository);
- $file = TMX . "{$repository}/{$locale}/cache_{$locale}.php";
+ $file = TMX . "{$locale}/cache_{$locale}_{$repository}.php";
if (! $tmx = Cache::getKey($file)) {
if (is_file($file)) {
View
@@ -278,7 +278,7 @@ function updateFromSVN() {
then
echogreen "Extract strings on svn"
cd $install
- nice -20 php app/inc/mozorg.php
+ nice -20 app/scripts/tmx_mozillaorg
fi
}
@@ -296,7 +296,7 @@ do
updateGaiaRepo ${gaia_version}
done
-# mozilla.org has its own extraction script, mozorg.php
+# mozilla.org has its own extraction script
updateFromSVN
# Generate productization data
View
@@ -30,7 +30,7 @@ eval $(cat $DIR/../config/config.ini | $DIR/ini_to_bash.py)
# Generate sources (gaia versions, supported locales)
echogreen "Generate list of locales and supported Gaia versions"
-php $DIR/generate_sources $config
+$DIR/generate_sources $config
# Check if we have sources
echogreen "Checking if Transvision sources are available..."
@@ -117,7 +117,7 @@ function checkoutSilme() {
then
echogreen "Checking out the SILME library into $libraries"
cd $libraries
- hg clone http://hg.mozilla.org/l10n/silme -u silme-0.8.0
+ hg clone https://hg.mozilla.org/l10n/silme -u silme-0.8.0
cd $install
fi
}
@@ -127,11 +127,11 @@ function initDesktopL10nRepo() {
if [ "$1" == "central" ]
then
local repo_folder="trunk_l10n"
- local repo_path="http://hg.mozilla.org/l10n-central"
+ local repo_path="https://hg.mozilla.org/l10n-central"
local locale_list="trunk_locales"
else
local repo_folder="${1}_l10n"
- local repo_path="http://hg.mozilla.org/releases/l10n/mozilla-$1"
+ local repo_path="https://hg.mozilla.org/releases/l10n/mozilla-$1"
local locale_list="${1}_locales"
fi
@@ -152,12 +152,11 @@ function initDesktopL10nRepo() {
hg clone $repo_path/$locale $locale
fi
- if [ ! -d $root/TMX/$1/$locale ]
+ if [ ! -d $root/TMX/$locale ]
then
# ${1^^} = uppercase($1)
- echogreen "Creating this locale cache for ${1^^}:"
- echogreen $locale
- mkdir -p $root/TMX/$1/$locale
+ echogreen "Creating folder cache for: $locale"
+ mkdir -p $root/TMX/$locale
fi
done
}
@@ -170,20 +169,20 @@ function initDesktopSourceRepo() {
echogreen "Checking out the following repo:"
echogreen $trunk_source/comm-central/
cd $trunk_source
- hg clone http://hg.mozilla.org/comm-central/
+ hg clone https://hg.mozilla.org/comm-central/
fi
if [ ! -d $trunk_source/mozilla-central/.hg ]
then
echogreen "Checking out the following repo:"
echogreen $trunk_source/mozilla-central/
cd $trunk_source
- hg clone http://hg.mozilla.org/mozilla-central/
+ hg clone https://hg.mozilla.org/mozilla-central/
fi
# Checkout chatzilla and venkman only on trunk, since they don't
# have branches. Can add other products to array nobranch_products,
- # as long as their code is located in http://hg.mozilla.org/PRODUCT
+ # as long as their code is located in https://hg.mozilla.org/PRODUCT
local nobranch_products=( chatzilla venkman )
for product in "${nobranch_products[@]}"
do
@@ -192,7 +191,7 @@ function initDesktopSourceRepo() {
echogreen "Checking out the following repo:"
echogreen $trunk_source/$product
cd $trunk_source
- hg clone http://hg.mozilla.org/$product/
+ hg clone https://hg.mozilla.org/$product/
fi
done
else
@@ -203,22 +202,21 @@ function initDesktopSourceRepo() {
then
echogreen "Checking out the following repo:"
echogreen ${!target}/comm-$1/
- hg clone http://hg.mozilla.org/releases/comm-$1/
+ hg clone https://hg.mozilla.org/releases/comm-$1/
fi
if [ ! -d ${!target}/mozilla-$1/.hg ]
then
echogreen "Checking out the following repo:"
echogreen ${!target}/mozilla-$1/
- hg clone http://hg.mozilla.org/releases/mozilla-$1/
+ hg clone https://hg.mozilla.org/releases/mozilla-$1/
fi
fi
- if [ ! -d $root/TMX/$1/en-US ]
+ if [ ! -d $root/TMX/en-US ]
then
- echogreen "Creating this locale cache for $1:"
- echogreen en-US
- mkdir -p $root/TMX/$1/en-US
+ echogreen "Creating folder cache for: en-US"
+ mkdir -p $root/TMX/en-US
fi
}
@@ -229,13 +227,13 @@ function initGaiaRepo () {
local locale_list="gaia_locales"
local repo_name="gaia"
local repo_pretty_name="Gaia"
- local repo_path="http://hg.mozilla.org/gaia-l10n"
+ local repo_path="https://hg.mozilla.org/gaia-l10n"
else
local locale_list="gaia_locales_$1"
local repo_name="gaia_$1"
# If version is "1_4", repo_pretty_name will be "Gaia 1.4"
local repo_pretty_name="Gaia ${1/_/.}"
- local repo_path="http://hg.mozilla.org/releases/gaia-l10n/v$1"
+ local repo_path="https://hg.mozilla.org/releases/gaia-l10n/v$1"
fi
echogreen "$repo_pretty_name initialization"
@@ -255,11 +253,10 @@ function initGaiaRepo () {
hg clone $repo_path/$locale
fi
- if [ ! -d $root/TMX/$repo_name/$locale ]
+ if [ ! -d $root/TMX/$locale ]
then
- echogreen "Creating this locale cache for $repo_pretty_name:"
- echogreen $locale
- mkdir -p $root/TMX/$repo_name/$locale
+ echogreen "Creating folder cache for: $locale"
+ mkdir -p $root/TMX/$locale
fi
done
fi
@@ -309,11 +306,10 @@ fi
for locale in $(cat $l20n_test_locales)
do
- if [ ! -d $root/TMX/l20n_test/$locale ]
+ if [ ! -d $root/TMX/$locale ]
then
- echogreen "Creating this locale cache for L20n test:"
- echogreen $locale
- mkdir -p $root/TMX/l20n_test/$locale
+ echogreen "Creating locale cache for: $locale"
+ mkdir -p $root/TMX/$locale
fi
done
View
@@ -1,3 +1,4 @@
+#!/usr/bin/env php
<?php
namespace Transvision;
@@ -6,7 +7,7 @@
die('Nope');
}
-include __DIR__ . '/init.php';
+include __DIR__ . '/../inc/init.php';
error_log('Mozilla.org: extraction of strings');
@@ -20,11 +21,14 @@ function ($item) use ($path) {
},
$mozilla_org_files);
- $out_english = $out_translation = '';
+ $out_translation = '';
$total_strings = 0;
+ // Check if this is the reference locale
+ $reference_locale = (Project::getReferenceLocale('mozilla_org') == $locale);
+
foreach ($mozilla_org_files as $file) {
- $strings = Dotlang::getStrings(SVN . "mozilla_org/{$locale}/{$file}");
+ $strings = Dotlang::getStrings(SVN . "mozilla_org/{$locale}/{$file}", $reference_locale);
$total_strings += count($strings);
foreach ($strings as $english => $translation) {
@@ -40,15 +44,12 @@ function ($item) use ($path) {
return $array_line;
};
- $out_english .= $output($english, $english);
$out_translation .= $output($english, $translation);
}
}
- $out_english = "<?php\n\$tmx = [\n" . $out_english . "];\n";
$out_translation = "<?php\n\$tmx = [\n" . $out_translation . "];\n";
- Files::fileForceContents(TMX . "mozilla_org/{$locale}/cache_{$locale}.php", $out_translation);
- Files::fileForceContents(TMX . "mozilla_org/{$locale}/cache_en-US.php", $out_english);
+ Files::fileForceContents(TMX . "{$locale}/cache_{$locale}_mozilla_org.php", $out_translation);
error_log("{$locale}: {$total_strings} strings");
}
View
@@ -1,9 +1,9 @@
#!/usr/bin/python
+import argparse
+import datetime
import os
import sys
-import datetime
-from optparse import OptionParser
from ConfigParser import SafeConfigParser
""" Here we read the server configuration file """
@@ -57,71 +57,65 @@ def php_close_array(target_file):
target_file.write("];\n")
if __name__ == "__main__":
- usage = "test"
- parser = OptionParser(usage, version='%prog 0.1')
- (options, args) = parser.parse_args(sys.argv[1:])
-
- locale_repo = args[0]
- en_US_repo = args[1]
- langcode1 = args[2]
- langcode2 = args[3]
- repo = args[4]
+ # Read command line input parameters
+ parser = argparse.ArgumentParser()
+ parser.add_argument('locale_repo', help='Path to locale files')
+ parser.add_argument('reference_repo', help='Path to reference files')
+ parser.add_argument('locale_code', help='Locale language code')
+ parser.add_argument('reference_code', help='Reference language code')
+ parser.add_argument('repository', help='Repository name')
+ args = parser.parse_args()
exclusionlist = ['.hgtags', '.hg', '.git', '.gitignore']
- dirs1 = os.listdir(locale_repo)
- if repo.startswith('gaia') or repo == 'l20n_test' :
- dirs2 = os.listdir(en_US_repo)
- dirs2 = [x for x in dirs2 if x not in exclusionlist]
+ dirs_locale = os.listdir(args.locale_repo)
+ if args.repository.startswith('gaia') or args.repository == 'l20n_test' :
+ dirs_reference = os.listdir(args.reference_repo)
+ dirs_reference = [x for x in dirs_reference if x not in exclusionlist]
else:
- dirs2 = ["browser", "calendar", "chat", "dom", "editor",
- "extensions", "mail", "mobile", "netwerk", "other-licenses",
- "security", "services", "suite", "toolkit", "webapprt"]
+ dirs_reference = [
+ "browser", "calendar", "chat", "dom", "editor",
+ "extensions", "mail", "mobile", "netwerk", "other-licenses",
+ "security", "services", "suite", "toolkit", "webapprt"
+ ]
- dirs = filter(lambda x:x in dirs1, dirs2)
+ dirs = filter(lambda x:x in dirs_locale, dirs_reference)
- localpath = os.path.join(localdir, repo, langcode1)
- filename1 = os.path.join(localpath, "cache_" + langcode2 + ".php")
- filename2 = os.path.join(localpath, "cache_" + langcode1 + ".php")
+ localpath = os.path.join(localdir, args.locale_code)
+ filename_locale = os.path.join(localpath, "cache_%s_%s.php" % (args.locale_code, args.repository))
- target_file1 = open(filename1, "w")
- target_file2 = open(filename2, "w")
- php_header(target_file1)
- php_header(target_file2)
+ target_locale = open(filename_locale, "w")
+ php_header(target_locale)
for directory in dirs:
-
- path1 = en_US_repo + directory
- path2 = locale_repo + directory
+ path_reference = args.reference_repo + directory
+ path_locale = args.locale_repo + directory
rcsClient = silme.io.Manager.get('file')
try:
- l10nPackage = rcsClient.get_package(path1, object_type='entitylist')
+ l10nPackage_reference = rcsClient.get_package(path_reference, object_type='entitylist')
except:
- print 'Silme couldn\'t extract data for ' + path1
+ print 'Silme couldn\'t extract data for ' + path_reference
continue
try:
- l10nPackage2 = rcsClient.get_package(path2, object_type='entitylist')
+ l10nPackage_locale = rcsClient.get_package(path_locale, object_type='entitylist')
except:
- print 'Silme couldn\'t extract data for ' + path2
+ print 'Silme couldn\'t extract data for ' + path_locale
continue
strings = {}
- chaine = get_string(l10nPackage, directory)
+ strings_reference = get_string(l10nPackage_reference, directory)
"""
get_string() is a recursive function that fills 'strings', a global array
We need to reset that global array before calling the function again
"""
del strings
strings = {}
- chaine2 = get_string(l10nPackage2, directory)
+ strings_locale = get_string(l10nPackage_locale, directory)
+ for entity in strings_reference:
+ php_add_to_array(entity, strings_locale.get(entity, ""), target_locale)
- for entity in chaine:
- php_add_to_array(entity, chaine[entity], target_file1)
- php_add_to_array(entity, chaine2.get(entity, ""), target_file2)
- php_close_array(target_file1)
- php_close_array(target_file2)
- target_file1.close()
- target_file2.close()
+ php_close_array(target_locale)
+ target_locale.close()
Oops, something went wrong.

0 comments on commit 91609fa

Please sign in to comment.