Skip to content
Permalink
Browse files

Restructure TMX folders

Move from /TMX/repo/locale to /TMX/locale

setup.sh
* Check existence of /TMX/locale
* Switch URLs to https

tmxmaker.py
* Use meaningful variable names
* Use argparse instead of OptionParser (deprecated since 2.7)
* Store cache files in TMX/locale, don't create duplicated en-US cache files

mozorg.php
* Move script from /app/inc/mozorg.php to /app/scripts/tmx_mozillaorg
* Generate cache file in /TMX/locale/cache_locale_mozilla_org.php

Utils::getRepoStrings
* Update cache file path and tests

Dotlang::getStrings
* Manage reference locale differently (store original string instead of empty)
  • Loading branch information
flodolo committed Jun 30, 2015
1 parent c79f342 commit 91609fa8c0b72f1c1317f866992087de134135b5
@@ -46,7 +46,7 @@ public static function getLangFilesList($path)
}

/**
* Loads the lang file and returns a cleaned up array of the lines
* Loads the lang file and returns a cleaned up array of the lines
*
* @param string $file path to the local file to load
* @return array All the significant lines in the file as an array
@@ -73,10 +73,11 @@ public static function getFile($file)
* ;String in english
* translated string
*
* @param string $file .lang file to load
* @return array Array of strings as [original => translation]
* @param string $file .lang file to load
* @param boolean $reference_locale True if the current locale is the reference locale
* @return array Array of strings as [original => translation]
*/
public static function getStrings($file)
public static function getStrings($file, $reference_locale)
{
// get all the lines in an array
$f = self::getFile($file);
@@ -104,7 +105,12 @@ public static function getStrings($file)
}

// If untranslated, I need to store an empty string as translation
$strings[$english] = ($translation == $english) ? '' : $translation;
// unless it's the reference locale
if ($reference_locale) {
$strings[$english] = $english;
} else {
$strings[$english] = ($translation == $english) ? '' : $translation;
}

$i++;
}
@@ -230,7 +230,7 @@ public static function getRepoStrings($locale, $repository)
{
$locale = Project::getLocaleInContext($locale, $repository);

$file = TMX . "{$repository}/{$locale}/cache_{$locale}.php";
$file = TMX . "{$locale}/cache_{$locale}_{$repository}.php";

if (! $tmx = Cache::getKey($file)) {
if (is_file($file)) {
@@ -278,7 +278,7 @@ function updateFromSVN() {
then
echogreen "Extract strings on svn"
cd $install
nice -20 php app/inc/mozorg.php
nice -20 app/scripts/tmx_mozillaorg
fi
}

@@ -296,7 +296,7 @@ do
updateGaiaRepo ${gaia_version}
done

# mozilla.org has its own extraction script, mozorg.php
# mozilla.org has its own extraction script
updateFromSVN

# Generate productization data
@@ -30,7 +30,7 @@ eval $(cat $DIR/../config/config.ini | $DIR/ini_to_bash.py)

# Generate sources (gaia versions, supported locales)
echogreen "Generate list of locales and supported Gaia versions"
php $DIR/generate_sources $config
$DIR/generate_sources $config

# Check if we have sources
echogreen "Checking if Transvision sources are available..."
@@ -117,7 +117,7 @@ function checkoutSilme() {
then
echogreen "Checking out the SILME library into $libraries"
cd $libraries
hg clone http://hg.mozilla.org/l10n/silme -u silme-0.8.0
hg clone https://hg.mozilla.org/l10n/silme -u silme-0.8.0
cd $install
fi
}
@@ -127,11 +127,11 @@ function initDesktopL10nRepo() {
if [ "$1" == "central" ]
then
local repo_folder="trunk_l10n"
local repo_path="http://hg.mozilla.org/l10n-central"
local repo_path="https://hg.mozilla.org/l10n-central"
local locale_list="trunk_locales"
else
local repo_folder="${1}_l10n"
local repo_path="http://hg.mozilla.org/releases/l10n/mozilla-$1"
local repo_path="https://hg.mozilla.org/releases/l10n/mozilla-$1"
local locale_list="${1}_locales"
fi

@@ -152,12 +152,11 @@ function initDesktopL10nRepo() {
hg clone $repo_path/$locale $locale
fi

if [ ! -d $root/TMX/$1/$locale ]
if [ ! -d $root/TMX/$locale ]
then
# ${1^^} = uppercase($1)
echogreen "Creating this locale cache for ${1^^}:"
echogreen $locale
mkdir -p $root/TMX/$1/$locale
echogreen "Creating folder cache for: $locale"
mkdir -p $root/TMX/$locale
fi
done
}
@@ -170,20 +169,20 @@ function initDesktopSourceRepo() {
echogreen "Checking out the following repo:"
echogreen $trunk_source/comm-central/
cd $trunk_source
hg clone http://hg.mozilla.org/comm-central/
hg clone https://hg.mozilla.org/comm-central/
fi

if [ ! -d $trunk_source/mozilla-central/.hg ]
then
echogreen "Checking out the following repo:"
echogreen $trunk_source/mozilla-central/
cd $trunk_source
hg clone http://hg.mozilla.org/mozilla-central/
hg clone https://hg.mozilla.org/mozilla-central/
fi

# Checkout chatzilla and venkman only on trunk, since they don't
# have branches. Can add other products to array nobranch_products,
# as long as their code is located in http://hg.mozilla.org/PRODUCT
# as long as their code is located in https://hg.mozilla.org/PRODUCT
local nobranch_products=( chatzilla venkman )
for product in "${nobranch_products[@]}"
do
@@ -192,7 +191,7 @@ function initDesktopSourceRepo() {
echogreen "Checking out the following repo:"
echogreen $trunk_source/$product
cd $trunk_source
hg clone http://hg.mozilla.org/$product/
hg clone https://hg.mozilla.org/$product/
fi
done
else
@@ -203,22 +202,21 @@ function initDesktopSourceRepo() {
then
echogreen "Checking out the following repo:"
echogreen ${!target}/comm-$1/
hg clone http://hg.mozilla.org/releases/comm-$1/
hg clone https://hg.mozilla.org/releases/comm-$1/
fi

if [ ! -d ${!target}/mozilla-$1/.hg ]
then
echogreen "Checking out the following repo:"
echogreen ${!target}/mozilla-$1/
hg clone http://hg.mozilla.org/releases/mozilla-$1/
hg clone https://hg.mozilla.org/releases/mozilla-$1/
fi
fi

if [ ! -d $root/TMX/$1/en-US ]
if [ ! -d $root/TMX/en-US ]
then
echogreen "Creating this locale cache for $1:"
echogreen en-US
mkdir -p $root/TMX/$1/en-US
echogreen "Creating folder cache for: en-US"
mkdir -p $root/TMX/en-US
fi
}

@@ -229,13 +227,13 @@ function initGaiaRepo () {
local locale_list="gaia_locales"
local repo_name="gaia"
local repo_pretty_name="Gaia"
local repo_path="http://hg.mozilla.org/gaia-l10n"
local repo_path="https://hg.mozilla.org/gaia-l10n"
else
local locale_list="gaia_locales_$1"
local repo_name="gaia_$1"
# If version is "1_4", repo_pretty_name will be "Gaia 1.4"
local repo_pretty_name="Gaia ${1/_/.}"
local repo_path="http://hg.mozilla.org/releases/gaia-l10n/v$1"
local repo_path="https://hg.mozilla.org/releases/gaia-l10n/v$1"
fi

echogreen "$repo_pretty_name initialization"
@@ -255,11 +253,10 @@ function initGaiaRepo () {
hg clone $repo_path/$locale
fi

if [ ! -d $root/TMX/$repo_name/$locale ]
if [ ! -d $root/TMX/$locale ]
then
echogreen "Creating this locale cache for $repo_pretty_name:"
echogreen $locale
mkdir -p $root/TMX/$repo_name/$locale
echogreen "Creating folder cache for: $locale"
mkdir -p $root/TMX/$locale
fi
done
fi
@@ -309,11 +306,10 @@ fi

for locale in $(cat $l20n_test_locales)
do
if [ ! -d $root/TMX/l20n_test/$locale ]
if [ ! -d $root/TMX/$locale ]
then
echogreen "Creating this locale cache for L20n test:"
echogreen $locale
mkdir -p $root/TMX/l20n_test/$locale
echogreen "Creating locale cache for: $locale"
mkdir -p $root/TMX/$locale
fi
done

@@ -1,3 +1,4 @@
#!/usr/bin/env php
<?php
namespace Transvision;

@@ -6,7 +7,7 @@
die('Nope');
}

include __DIR__ . '/init.php';
include __DIR__ . '/../inc/init.php';

error_log('Mozilla.org: extraction of strings');

@@ -20,11 +21,14 @@ function ($item) use ($path) {
},
$mozilla_org_files);

$out_english = $out_translation = '';
$out_translation = '';
$total_strings = 0;

// Check if this is the reference locale
$reference_locale = (Project::getReferenceLocale('mozilla_org') == $locale);

foreach ($mozilla_org_files as $file) {
$strings = Dotlang::getStrings(SVN . "mozilla_org/{$locale}/{$file}");
$strings = Dotlang::getStrings(SVN . "mozilla_org/{$locale}/{$file}", $reference_locale);
$total_strings += count($strings);

foreach ($strings as $english => $translation) {
@@ -40,15 +44,12 @@ function ($item) use ($path) {
return $array_line;
};

$out_english .= $output($english, $english);
$out_translation .= $output($english, $translation);
}
}

$out_english = "<?php\n\$tmx = [\n" . $out_english . "];\n";
$out_translation = "<?php\n\$tmx = [\n" . $out_translation . "];\n";

Files::fileForceContents(TMX . "mozilla_org/{$locale}/cache_{$locale}.php", $out_translation);
Files::fileForceContents(TMX . "mozilla_org/{$locale}/cache_en-US.php", $out_english);
Files::fileForceContents(TMX . "{$locale}/cache_{$locale}_mozilla_org.php", $out_translation);
error_log("{$locale}: {$total_strings} strings");
}
@@ -1,9 +1,9 @@
#!/usr/bin/python

import argparse
import datetime
import os
import sys
import datetime
from optparse import OptionParser
from ConfigParser import SafeConfigParser

""" Here we read the server configuration file """
@@ -57,71 +57,65 @@ def php_close_array(target_file):
target_file.write("];\n")

if __name__ == "__main__":
usage = "test"
parser = OptionParser(usage, version='%prog 0.1')
(options, args) = parser.parse_args(sys.argv[1:])

locale_repo = args[0]
en_US_repo = args[1]
langcode1 = args[2]
langcode2 = args[3]
repo = args[4]
# Read command line input parameters
parser = argparse.ArgumentParser()
parser.add_argument('locale_repo', help='Path to locale files')
parser.add_argument('reference_repo', help='Path to reference files')
parser.add_argument('locale_code', help='Locale language code')
parser.add_argument('reference_code', help='Reference language code')
parser.add_argument('repository', help='Repository name')
args = parser.parse_args()

exclusionlist = ['.hgtags', '.hg', '.git', '.gitignore']
dirs1 = os.listdir(locale_repo)
if repo.startswith('gaia') or repo == 'l20n_test' :
dirs2 = os.listdir(en_US_repo)
dirs2 = [x for x in dirs2 if x not in exclusionlist]
dirs_locale = os.listdir(args.locale_repo)
if args.repository.startswith('gaia') or args.repository == 'l20n_test' :
dirs_reference = os.listdir(args.reference_repo)
dirs_reference = [x for x in dirs_reference if x not in exclusionlist]
else:
dirs2 = ["browser", "calendar", "chat", "dom", "editor",
"extensions", "mail", "mobile", "netwerk", "other-licenses",
"security", "services", "suite", "toolkit", "webapprt"]
dirs_reference = [
"browser", "calendar", "chat", "dom", "editor",
"extensions", "mail", "mobile", "netwerk", "other-licenses",
"security", "services", "suite", "toolkit", "webapprt"
]

dirs = filter(lambda x:x in dirs1, dirs2)
dirs = filter(lambda x:x in dirs_locale, dirs_reference)

localpath = os.path.join(localdir, repo, langcode1)
filename1 = os.path.join(localpath, "cache_" + langcode2 + ".php")
filename2 = os.path.join(localpath, "cache_" + langcode1 + ".php")
localpath = os.path.join(localdir, args.locale_code)
filename_locale = os.path.join(localpath, "cache_%s_%s.php" % (args.locale_code, args.repository))

target_file1 = open(filename1, "w")
target_file2 = open(filename2, "w")
php_header(target_file1)
php_header(target_file2)
target_locale = open(filename_locale, "w")
php_header(target_locale)

for directory in dirs:

path1 = en_US_repo + directory
path2 = locale_repo + directory
path_reference = args.reference_repo + directory
path_locale = args.locale_repo + directory

rcsClient = silme.io.Manager.get('file')
try:
l10nPackage = rcsClient.get_package(path1, object_type='entitylist')
l10nPackage_reference = rcsClient.get_package(path_reference, object_type='entitylist')
except:
print 'Silme couldn\'t extract data for ' + path1
print 'Silme couldn\'t extract data for ' + path_reference
continue

try:
l10nPackage2 = rcsClient.get_package(path2, object_type='entitylist')
l10nPackage_locale = rcsClient.get_package(path_locale, object_type='entitylist')
except:
print 'Silme couldn\'t extract data for ' + path2
print 'Silme couldn\'t extract data for ' + path_locale
continue

strings = {}
chaine = get_string(l10nPackage, directory)
strings_reference = get_string(l10nPackage_reference, directory)

"""
get_string() is a recursive function that fills 'strings', a global array
We need to reset that global array before calling the function again
"""
del strings
strings = {}
chaine2 = get_string(l10nPackage2, directory)
strings_locale = get_string(l10nPackage_locale, directory)

for entity in strings_reference:
php_add_to_array(entity, strings_locale.get(entity, ""), target_locale)

for entity in chaine:
php_add_to_array(entity, chaine[entity], target_file1)
php_add_to_array(entity, chaine2.get(entity, ""), target_file2)
php_close_array(target_file1)
php_close_array(target_file2)
target_file1.close()
target_file2.close()
php_close_array(target_locale)
target_locale.close()
File renamed without changes.
File renamed without changes.

0 comments on commit 91609fa

Please sign in to comment.
You can’t perform that action at this time.