Restructure TMX folders

Move from /TMX/repo/locale to /TMX/locale setup.sh * Check existence of /TMX/locale * Switch URLs to https tmxmaker.py * Use meaningful variable names * Use argparse instead of OptionParser (deprecated since 2.7) * Store cache files in TMX/locale, don't create duplicated en-US cache files mozorg.php * Move script from /app/inc/mozorg.php to /app/scripts/tmx_mozillaorg * Generate cache file in /TMX/locale/cache_locale_mozilla_org.php Utils::getRepoStrings * Update cache file path and tests Dotlang::getStrings * Manage reference locale differently (store original string instead of empty)
mozfr · Jun 30, 2015 · 91609fa · 91609fa
1 parent c79f342
commit 91609fa
Show file tree

Hide file tree

Showing 12 changed files with 101 additions and 93 deletions.
diff --git a/app/classes/Transvision/Dotlang.php b/app/classes/Transvision/Dotlang.php
@@ -46,7 +46,7 @@ public static function getLangFilesList($path)
     }
 
     /**
-     * Loads the  lang file and returns a cleaned up array of the lines
+     * Loads the lang file and returns a cleaned up array of the lines
      *
      * @param  string $file path to the local file to load
      * @return array  All the significant lines in the file as an array
@@ -73,10 +73,11 @@ public static function getFile($file)
      *  ;String in english
      *  translated string
      *
-     * @param  string $file .lang file to load
-     * @return array  Array of strings as [original => translation]
+     * @param  string  $file             .lang file to load
+     * @param  boolean $reference_locale True if the current locale is the reference locale
+     * @return array   Array of strings as [original => translation]
      */
-    public static function getStrings($file)
+    public static function getStrings($file, $reference_locale)
     {
         // get all the lines in an array
         $f = self::getFile($file);
@@ -104,7 +105,12 @@ public static function getStrings($file)
                 }
 
                 // If untranslated, I need to store an empty string as translation
-                $strings[$english] = ($translation == $english) ? '' : $translation;
+                // unless it's the reference locale
+                if ($reference_locale) {
+                    $strings[$english] = $english;
+                } else {
+                    $strings[$english] = ($translation == $english) ? '' : $translation;
+                }
 
                 $i++;
             }

diff --git a/app/classes/Transvision/Utils.php b/app/classes/Transvision/Utils.php
@@ -230,7 +230,7 @@ public static function getRepoStrings($locale, $repository)
     {
         $locale = Project::getLocaleInContext($locale, $repository);
 
-        $file = TMX . "{$repository}/{$locale}/cache_{$locale}.php";
+        $file = TMX . "{$locale}/cache_{$locale}_{$repository}.php";
 
         if (! $tmx = Cache::getKey($file)) {
             if (is_file($file)) {

diff --git a/app/scripts/glossaire.sh b/app/scripts/glossaire.sh
@@ -278,7 +278,7 @@ function updateFromSVN() {
     then
         echogreen "Extract strings on svn"
         cd $install
-        nice -20 php app/inc/mozorg.php
+        nice -20 app/scripts/tmx_mozillaorg
     fi
 }
 
@@ -296,7 +296,7 @@ do
     updateGaiaRepo ${gaia_version}
 done
 
-# mozilla.org has its own extraction script, mozorg.php
+# mozilla.org has its own extraction script
 updateFromSVN
 
 # Generate productization data

diff --git a/app/scripts/setup.sh b/app/scripts/setup.sh
@@ -30,7 +30,7 @@ eval $(cat $DIR/../config/config.ini | $DIR/ini_to_bash.py)
 
 # Generate sources (gaia versions, supported locales)
 echogreen "Generate list of locales and supported Gaia versions"
-php $DIR/generate_sources $config
+$DIR/generate_sources $config
 
 # Check if we have sources
 echogreen "Checking if Transvision sources are available..."
@@ -117,7 +117,7 @@ function checkoutSilme() {
     then
         echogreen "Checking out the SILME library into $libraries"
         cd $libraries
-        hg clone http://hg.mozilla.org/l10n/silme -u silme-0.8.0
+        hg clone https://hg.mozilla.org/l10n/silme -u silme-0.8.0
         cd $install
     fi
 }
@@ -127,11 +127,11 @@ function initDesktopL10nRepo() {
     if [ "$1" == "central" ]
     then
         local repo_folder="trunk_l10n"
-        local repo_path="http://hg.mozilla.org/l10n-central"
+        local repo_path="https://hg.mozilla.org/l10n-central"
         local locale_list="trunk_locales"
     else
         local repo_folder="${1}_l10n"
-        local repo_path="http://hg.mozilla.org/releases/l10n/mozilla-$1"
+        local repo_path="https://hg.mozilla.org/releases/l10n/mozilla-$1"
         local locale_list="${1}_locales"
     fi
 
@@ -152,12 +152,11 @@ function initDesktopL10nRepo() {
                 hg clone $repo_path/$locale $locale
             fi
 
-            if [ ! -d $root/TMX/$1/$locale ]
+            if [ ! -d $root/TMX/$locale ]
             then
                 # ${1^^} = uppercase($1)
-                echogreen "Creating this locale cache for ${1^^}:"
-                echogreen $locale
-                mkdir -p $root/TMX/$1/$locale
+                echogreen "Creating folder cache for: $locale"
+                mkdir -p $root/TMX/$locale
             fi
     done
 }
@@ -170,20 +169,20 @@ function initDesktopSourceRepo() {
             echogreen "Checking out the following repo:"
             echogreen $trunk_source/comm-central/
             cd $trunk_source
-            hg clone http://hg.mozilla.org/comm-central/
+            hg clone https://hg.mozilla.org/comm-central/
         fi
 
         if [ ! -d $trunk_source/mozilla-central/.hg ]
         then
             echogreen "Checking out the following repo:"
             echogreen $trunk_source/mozilla-central/
             cd $trunk_source
-            hg clone http://hg.mozilla.org/mozilla-central/
+            hg clone https://hg.mozilla.org/mozilla-central/
         fi
 
         # Checkout chatzilla and venkman only on trunk, since they don't
         # have branches. Can add other products to array nobranch_products,
-        # as long as their code is located in http://hg.mozilla.org/PRODUCT
+        # as long as their code is located in https://hg.mozilla.org/PRODUCT
         local nobranch_products=( chatzilla venkman )
         for product in "${nobranch_products[@]}"
         do
@@ -192,7 +191,7 @@ function initDesktopSourceRepo() {
                 echogreen "Checking out the following repo:"
                 echogreen $trunk_source/$product
                 cd $trunk_source
-                hg clone http://hg.mozilla.org/$product/
+                hg clone https://hg.mozilla.org/$product/
             fi
         done
     else
@@ -203,22 +202,21 @@ function initDesktopSourceRepo() {
         then
             echogreen "Checking out the following repo:"
             echogreen ${!target}/comm-$1/
-            hg clone http://hg.mozilla.org/releases/comm-$1/
+            hg clone https://hg.mozilla.org/releases/comm-$1/
         fi
 
         if [ ! -d ${!target}/mozilla-$1/.hg ]
         then
             echogreen "Checking out the following repo:"
             echogreen ${!target}/mozilla-$1/
-            hg clone http://hg.mozilla.org/releases/mozilla-$1/
+            hg clone https://hg.mozilla.org/releases/mozilla-$1/
         fi
     fi
 
-    if [ ! -d $root/TMX/$1/en-US ]
+    if [ ! -d $root/TMX/en-US ]
     then
-        echogreen "Creating this locale cache for $1:"
-        echogreen en-US
-        mkdir -p $root/TMX/$1/en-US
+        echogreen "Creating folder cache for: en-US"
+        mkdir -p $root/TMX/en-US
     fi
 }
 
@@ -229,13 +227,13 @@ function initGaiaRepo () {
         local locale_list="gaia_locales"
         local repo_name="gaia"
         local repo_pretty_name="Gaia"
-        local repo_path="http://hg.mozilla.org/gaia-l10n"
+        local repo_path="https://hg.mozilla.org/gaia-l10n"
     else
         local locale_list="gaia_locales_$1"
         local repo_name="gaia_$1"
         # If version is "1_4", repo_pretty_name will be "Gaia 1.4"
         local repo_pretty_name="Gaia ${1/_/.}"
-        local repo_path="http://hg.mozilla.org/releases/gaia-l10n/v$1"
+        local repo_path="https://hg.mozilla.org/releases/gaia-l10n/v$1"
     fi
 
     echogreen "$repo_pretty_name initialization"
@@ -255,11 +253,10 @@ function initGaiaRepo () {
                     hg clone $repo_path/$locale
                 fi
 
-                if [ ! -d $root/TMX/$repo_name/$locale ]
+                if [ ! -d $root/TMX/$locale ]
                 then
-                    echogreen "Creating this locale cache for $repo_pretty_name:"
-                    echogreen $locale
-                    mkdir -p $root/TMX/$repo_name/$locale
+                    echogreen "Creating folder cache for: $locale"
+                    mkdir -p $root/TMX/$locale
                 fi
         done
     fi
@@ -309,11 +306,10 @@ fi
 
 for locale in $(cat $l20n_test_locales)
     do
-        if [ ! -d $root/TMX/l20n_test/$locale ]
+        if [ ! -d $root/TMX/$locale ]
         then
-            echogreen "Creating this locale cache for L20n test:"
-            echogreen $locale
-            mkdir -p $root/TMX/l20n_test/$locale
+            echogreen "Creating locale cache for: $locale"
+            mkdir -p $root/TMX/$locale
         fi
 done
 

diff --git a/app/inc/mozorg.php → app/scripts/tmx_mozillaorg b/app/inc/mozorg.php → app/scripts/tmx_mozillaorg
@@ -1,3 +1,4 @@
+#!/usr/bin/env php
 <?php
 namespace Transvision;
 
@@ -6,7 +7,7 @@
     die('Nope');
 }
 
-include __DIR__ . '/init.php';
+include __DIR__ . '/../inc/init.php';
 
 error_log('Mozilla.org: extraction of strings');
 
@@ -20,11 +21,14 @@ function ($item) use ($path) {
         },
         $mozilla_org_files);
 
-    $out_english = $out_translation = '';
+    $out_translation = '';
     $total_strings = 0;
 
+    // Check if this is the reference locale
+    $reference_locale = (Project::getReferenceLocale('mozilla_org') == $locale);
+
     foreach ($mozilla_org_files as $file) {
-        $strings = Dotlang::getStrings(SVN . "mozilla_org/{$locale}/{$file}");
+        $strings = Dotlang::getStrings(SVN . "mozilla_org/{$locale}/{$file}", $reference_locale);
         $total_strings += count($strings);
 
         foreach ($strings as $english => $translation) {
@@ -40,15 +44,12 @@ function ($item) use ($path) {
                 return $array_line;
             };
 
-            $out_english     .= $output($english, $english);
             $out_translation .= $output($english, $translation);
         }
     }
 
-    $out_english     = "<?php\n\$tmx = [\n" . $out_english .  "];\n";
     $out_translation = "<?php\n\$tmx = [\n" . $out_translation .  "];\n";
 
-    Files::fileForceContents(TMX . "mozilla_org/{$locale}/cache_{$locale}.php", $out_translation);
-    Files::fileForceContents(TMX . "mozilla_org/{$locale}/cache_en-US.php", $out_english);
+    Files::fileForceContents(TMX . "{$locale}/cache_{$locale}_mozilla_org.php", $out_translation);
     error_log("{$locale}: {$total_strings} strings");
 }
diff --git a/app/scripts/tmxmaker.py b/app/scripts/tmxmaker.py
@@ -1,9 +1,9 @@
 #!/usr/bin/python
 
+import argparse
+import datetime
 import os
 import sys
-import datetime
-from optparse import OptionParser
 from ConfigParser import SafeConfigParser
 
 """ Here we read the server configuration file """
@@ -57,71 +57,65 @@ def php_close_array(target_file):
     target_file.write("];\n")
 
 if __name__ == "__main__":
-    usage = "test"
-    parser = OptionParser(usage, version='%prog 0.1')
-    (options, args) = parser.parse_args(sys.argv[1:])
-
-    locale_repo = args[0]
-    en_US_repo = args[1]
-    langcode1 = args[2]
-    langcode2 = args[3]
-    repo = args[4]
+    # Read command line input parameters
+    parser = argparse.ArgumentParser()
+    parser.add_argument('locale_repo', help='Path to locale files')
+    parser.add_argument('reference_repo', help='Path to reference files')
+    parser.add_argument('locale_code', help='Locale language code')
+    parser.add_argument('reference_code', help='Reference language code')
+    parser.add_argument('repository', help='Repository name')
+    args = parser.parse_args()
 
     exclusionlist = ['.hgtags', '.hg', '.git', '.gitignore']
-    dirs1 = os.listdir(locale_repo)
-    if repo.startswith('gaia') or repo == 'l20n_test' :
-        dirs2 = os.listdir(en_US_repo)
-        dirs2 = [x for x in dirs2 if x not in exclusionlist]
+    dirs_locale = os.listdir(args.locale_repo)
+    if args.repository.startswith('gaia') or args.repository == 'l20n_test' :
+        dirs_reference = os.listdir(args.reference_repo)
+        dirs_reference = [x for x in dirs_reference if x not in exclusionlist]
     else:
-        dirs2 = ["browser", "calendar", "chat", "dom", "editor",
-                "extensions", "mail", "mobile", "netwerk", "other-licenses",
-                "security", "services", "suite", "toolkit", "webapprt"]
+        dirs_reference = [
+            "browser", "calendar", "chat", "dom", "editor",
+            "extensions", "mail", "mobile", "netwerk", "other-licenses",
+            "security", "services", "suite", "toolkit", "webapprt"
+        ]
 
-    dirs = filter(lambda x:x in dirs1, dirs2)
+    dirs = filter(lambda x:x in dirs_locale, dirs_reference)
 
-    localpath = os.path.join(localdir, repo, langcode1)
-    filename1 = os.path.join(localpath, "cache_" + langcode2 + ".php")
-    filename2 = os.path.join(localpath, "cache_" + langcode1 + ".php")
+    localpath = os.path.join(localdir, args.locale_code)
+    filename_locale = os.path.join(localpath, "cache_%s_%s.php" % (args.locale_code, args.repository))
 
-    target_file1 = open(filename1, "w")
-    target_file2 = open(filename2, "w")
-    php_header(target_file1)
-    php_header(target_file2)
+    target_locale = open(filename_locale, "w")
+    php_header(target_locale)
 
     for directory in dirs:
-
-        path1 = en_US_repo + directory
-        path2 = locale_repo + directory
+        path_reference = args.reference_repo + directory
+        path_locale = args.locale_repo + directory
 
         rcsClient = silme.io.Manager.get('file')
         try:
-            l10nPackage = rcsClient.get_package(path1, object_type='entitylist')
+            l10nPackage_reference = rcsClient.get_package(path_reference, object_type='entitylist')
         except:
-            print 'Silme couldn\'t extract data for ' + path1
+            print 'Silme couldn\'t extract data for ' + path_reference
             continue
 
         try:
-            l10nPackage2 = rcsClient.get_package(path2, object_type='entitylist')
+            l10nPackage_locale = rcsClient.get_package(path_locale, object_type='entitylist')
         except:
-            print 'Silme couldn\'t extract data for ' + path2
+            print 'Silme couldn\'t extract data for ' + path_locale
             continue
 
         strings = {}
-        chaine = get_string(l10nPackage, directory)
+        strings_reference = get_string(l10nPackage_reference, directory)
 
         """
         get_string() is a recursive function that fills 'strings', a global array
         We need to reset that global array before calling the function again
         """
         del strings
         strings = {}
-        chaine2 = get_string(l10nPackage2, directory)
+        strings_locale = get_string(l10nPackage_locale, directory)
 
+        for entity in strings_reference:
+            php_add_to_array(entity, strings_locale.get(entity, ""), target_locale)
 
-        for entity in chaine:
-            php_add_to_array(entity, chaine[entity], target_file1)
-            php_add_to_array(entity, chaine2.get(entity, ""), target_file2)
-    php_close_array(target_file1)
-    php_close_array(target_file2)
-    target_file1.close()
-    target_file2.close()
+    php_close_array(target_locale)
+    target_locale.close()
diff --git a/...stfiles/TMX/central/en-US/cache_en-US.php → ...stfiles/TMX/en-US/cache_en-US_central.php b/...stfiles/TMX/central/en-US/cache_en-US.php → ...stfiles/TMX/en-US/cache_en-US_central.php
diff --git a/tests/testfiles/TMX/gaia/es/cache_es.php → tests/testfiles/TMX/es/cache_es_gaia.php b/tests/testfiles/TMX/gaia/es/cache_es.php → tests/testfiles/TMX/es/cache_es_gaia.php
diff --git a/tests/testfiles/TMX/central/fr/cache_fr.php → tests/testfiles/TMX/fr/cache_fr_central.php b/tests/testfiles/TMX/central/fr/cache_fr.php → tests/testfiles/TMX/fr/cache_fr_central.php