Improve support for VPN and XLIFF (#1003)

* Consider underscores vs hyphen when extracting XLIFF
* Support underscores in VCS links
* Fix Pontoon edit link for VPN client
* XLIFF: Don't consider missing strings as empty strings
* Ignore folders in clean-up script for VPN

Author: flodolo

app/classes/Transvision/Project.php

@@ -38,6 +38,9 @@ class Project
      *
      * locale_mapping: if locale codes need to be mapped (Mozilla code -> Repo code)
      *
+     * underscore_locale: set to true if using underscores as separator (en_CA)
+     *                    instead of hyphens (en-CA)
+     *
      * pontoon_project: name of the project in Pontoon
      *
      * source_type: source type used by the project (xliff, gettext, etc.)
@@ -83,14 +86,15 @@ class Project
             'variable_patterns' => ['xml_android'],
         ],
         'vpn_client' => [
-            'files'             => [
+            'files'              => [
                 'mozillavpn.xliff',
             ],
-            'git_repository'    => 'mozilla-vpn-client-l10n',
-            'pontoon_project'   => 'mozilla-vpn-client',
-            'reference_locale'  => 'en',
-            'source_type'       => 'xliff',
-            'variable_patterns' => ['xliff-qt'],
+            'git_repository'     => 'mozilla-vpn-client-l10n',
+            'underscore_locales' => true,
+            'pontoon_project'    => 'mozilla-vpn-client',
+            'reference_locale'   => 'en',
+            'source_type'        => 'xliff',
+            'variable_patterns'  => ['xliff-qt'],
         ],
     ];
 
@@ -110,6 +114,7 @@ class Project
         // Products using free text search on Pontoon
         'text_search' => [
             'firefox_ios',
+            'vpn_client',
         ],
     ];
 

app/classes/Transvision/VersionControl.php

@@ -61,6 +61,11 @@ public static function getPath($locale, $repo, $path)
     {
         $vcs = self::getVCS($repo);
 
+        $repo_data = Project::$repos_info[$repo];
+        $locale = isset($repo_data['underscore_locales']) && $repo_data['underscore_locales']
+            ? str_replace('-', '_', $locale)
+            : $locale;
+
         switch ($vcs) {
             case 'git':
                 $path = self::gitPath($locale, $repo, $path);

app/classes/Transvision/Xliff.php

@@ -37,12 +37,20 @@ public static function getStrings($xliff_path, $project_name, $reference_locale
                 $file_orig = $file_node[0]['original'];
 
                 $string_id = self::generateStringID($project_name, $file_name, $file_orig, $trans_unit['id']);
-                // If it's the reference locale, we use the source instead of the target
-                $translation = $reference_locale
-                    ? str_replace("'", "\\'", $trans_unit->source)
-                    : str_replace("'", "\\'", $trans_unit->target);
 
-                $strings[$string_id] = $translation;
+                if ($reference_locale) {
+                    // If it's the reference locale, we use the source instead of the target
+                    $translation = str_replace("'", "\\'", $trans_unit->source);
+                    $strings[$string_id] = $translation;
+                } elseif (isset($trans_unit->target)) {
+                    /*
+                        We only store the translation if the target is set.
+                        simplexml returns an empty string if the element is
+                        missing.
+                    */
+                    $translation = str_replace("'", "\\'", $trans_unit->target);
+                    $strings[$string_id] = $translation;
+                }
             }
         }
 

app/scripts/clean_data.py

@@ -94,6 +94,7 @@ def main():
     excluded_folders = {
         "firefox_ios": [".git", "templates"],
         "mozilla_org": [".git", "configs", "en"],
+        "vpn_client": [".git", ".github", "en"],
     }
 
     """

app/scripts/tmx/tmx_xliff

@@ -36,12 +36,17 @@ error_log('Extraction of strings from XLIFF file');
 $base_path = isset($repo_data['git_subfolder'])
     ? GIT . "{$project_name}/{$repo_data['git_subfolder']}"
     : GIT . $project_name;
-foreach (Files::getFoldersInPath($base_path, ['templates']) as $locale) {
+foreach (Files::getFoldersInPath($base_path, ['templates']) as $locale_folder) {
     $out_translation = '';
     $total_strings = 0;
 
+    # Always use hyphens for locale codes
+    $locale = isset($repo_data['underscore_locales']) && $repo_data['underscore_locales']
+        ? str_replace('_', '-', $locale_folder)
+        : $locale_folder;
+
     foreach ($repo_data['files'] as $file_name) {
-        $xliff_path = "{$base_path}/{$locale}/{$file_name}";
+        $xliff_path = "{$base_path}/{$locale_folder}/{$file_name}";
         if (file_exists($xliff_path)) {
             // Make sure there is a reference locale defined
             $reference_locale = isset($repo_data['reference_locale'])

tests/testfiles/xliff/firefox-ios.xliff

@@ -86,7 +86,6 @@
       </trans-unit>
       <trans-unit id="Cookies">
         <source>Cookies</source>
-        <target>Cookie</target>
       </trans-unit>
     </body>
   </file>

tests/units/Transvision/VersionControl.php

@@ -163,6 +163,12 @@ public function getPathDP()
                 'firefox_ios/firefox-ios.xliff:0f4d892c',
                 'https://github.com/mozilla-l10n/firefoxios-l10n/blob/master/it/firefox-ios.xliff',
             ],
+            [
+                'sv-SE',
+                'vpn_client',
+                'vpn_client/mozillavpn.xliff.xliff:0f4d892c',
+                'https://github.com/mozilla-l10n/mozilla-vpn-client-l10n/blob/master/sv_SE/mozillavpn.xliff.xliff',
+            ],
         ];
     }
 

tests/units/Transvision/Xliff.php

@@ -16,7 +16,7 @@ public function testGetStrings()
         // Check total number of strings
         $this
             ->integer(count($strings))
-                ->isEqualTo(14);
+                ->isEqualTo(13);
 
         // Check strings
         $this