Support comm-l10n and search across all projects (#1021)

* Support search across all projects in main UI (it was already supported via API)
* Run mozilla-l10n-query locally during setup instead of depending on an external service
* Add Thunderbird as a separate project, remove Chatzilla

### Other fixes

* Don't show Bugzilla link on reference locale
* Remove t2t view
* Switch repository from `master` to `main`

Author: flodolo

.github/workflows/tests.yaml

@@ -1,13 +1,13 @@
 name: Run Tests
 on:
-  # Trigger on push to master
+  # Trigger on push to the default branch
   push:
     branches:
-      - master
+      - main
   # Trigger the workflow on pull request events
-  # but only for the master branch
+  # but only for the default branch
   pull_request:
-    branches: [ master ]
+    branches: [ main ]
     types: [ opened, synchronize, reopened ]
   workflow_dispatch:
 jobs:
@@ -58,6 +58,7 @@ jobs:
         run: |
           cp app/config/config.ini-ghactions app/config/config.ini
           vendor/atoum/atoum/bin/atoum -d tests/units/ --use-light-report
+
       - name: Functional tests (API, pages)
         run: |
           php tests/functional/api.php

.php-cs-fixer.dist.php

@@ -36,7 +36,7 @@
             'no_leading_import_slash'                    => true,
             'no_leading_namespace_whitespace'            => true,
             'no_singleline_whitespace_before_semicolons' => true,
-            'no_trailing_comma_in_singleline_array'      => true,
+            'no_trailing_comma_in_singleline'            => true,
             'no_unused_imports'                          => true,
             'no_whitespace_in_blank_line'                => true,
             'object_operator_without_whitespace'         => true,

CONTRIBUTING.md

@@ -38,10 +38,10 @@ php-cs-fixer will tell you if it changed some of the files in the repository, do
 ```bash
 git remote add upstream git@github.com:mozfr/transvision.git
 ```
-- Update your master branch to the latest version of Transvision every time you want to do some dev work:
+- Update your default branch to the latest version of Transvision every time you want to do some dev work:
 ```bash
-git checkout master
-git pull upstream master
+git checkout main
+git pull upstream main
 ```
 Then switch to a new branch where you will work on the patch you want to propose:
 ```bash

app/classes/Transvision/API.php

@@ -231,8 +231,7 @@ private function isValidServiceCall($service)
                 */
                 if (isset($this->parameters[2])) {
                     $match = false;
-
-                    foreach (Project::getRepositories() as $repository) {
+                    foreach (Project::getRepositories(true) as $repository) {
                         if ($this->verifyLocaleExists($this->parameters[2], $repository)) {
                             $match = true;
                             break;
@@ -354,8 +353,8 @@ private function verifyEnoughParameters($number)
     private function verifyRepositoryExists($repository, $alias = false)
     {
         $this->valid_repositories = $alias
-            ? array_merge(Project::getRepositories(), ['global'])
-            : Project::getRepositories();
+            ? array_merge(Project::getRepositories(true), ['global'])
+            : Project::getRepositories(true);
 
         if (! in_array($repository, $this->valid_repositories)) {
             $this->log("The repo queried ({$repository}) doesn't exist.");

app/classes/Transvision/Project.php

@@ -13,18 +13,6 @@
  */
 class Project
 {
-    /**
-     * This array maps different subfolders name for Desktop products
-     * with their display name
-     */
-    public static $components_names = [
-        'browser'  => 'Firefox Desktop',
-        'mobile'   => 'Firefox for Android',
-        'mail'     => 'Thunderbird',
-        'suite'    => 'SeaMonkey',
-        'calendar' => 'Lightning',
-    ];
-
     /*
      * This array contains information about supported repositories.
      *
@@ -52,6 +40,9 @@ class Project
      *
      */
     public static $repos_info = [
+        'all_projects' => [
+            'meta' => true,
+        ],
         'firefox_ios' => [
             'git_repository'    => 'firefoxios-l10n',
             'locale_mapping'    => [
@@ -68,15 +59,21 @@ class Project
             'source_type'       => 'mixed',
             'variable_patterns' => ['dtd', 'ftl', 'l10njs', 'printf', 'properties'],
         ],
+        'comm_l10n'=> [
+            'source_type'       => 'mixed',
+            'variable_patterns' => ['dtd', 'ftl', 'l10njs', 'printf', 'properties'],
+        ],
         'mozilla_org'=> [
             'git_repository'    => 'www-l10n',
+            'git_branch'        => 'master',
             'pontoon_project'   => 'mozillaorg',
             'reference_locale'  => 'en',
             'source_type'       => 'mixed',
             'variable_patterns' => ['ftl'],
         ],
         'android_l10n' => [
             'git_repository'    => 'android-l10n',
+            'git_branch'        => 'master',
             'locale_mapping'    => [], // To avoid using Bugzilla
             'pontoon_project'   => 'android-l10n',
             'source_type'       => 'xml',
@@ -100,6 +97,7 @@ class Project
         // Desktop products
         'desktop'     => [
             'gecko_strings',
+            'comm_l10n',
         ],
         // Products using Git
         'git' => [
@@ -115,10 +113,12 @@ class Project
     /**
      * Create a list of all supported repositories.
      *
+     * @param boolean $exclude_meta Exclude meta project
+     *
      * @return array List of supported repositories, key is the repo, value is
      *               the nice name for the repo for display purposes.
      */
-    public static function getSupportedRepositories()
+    public static function getSupportedRepositories($exclude_meta = false)
     {
         // Read list of repositories from supported_repositories.json
         $file_name = APP_SOURCES . 'supported_repositories.json';
@@ -130,6 +130,9 @@ public static function getSupportedRepositories()
 
         $repositories = [];
         foreach ($json_repositories as $repository) {
+            if ($exclude_meta && self::isMetaRepository($repository['id'])) {
+                continue;
+            }
             $repositories[$repository['id']] = $repository['name'];
         }
 
@@ -139,11 +142,13 @@ public static function getSupportedRepositories()
     /**
      * Get the list of repositories.
      *
+     * @param boolean $exclude_meta Exclude meta projects
+     *
      * @return array list of local repositories values
      */
-    public static function getRepositories()
+    public static function getRepositories($exclude_meta = false)
     {
-        $repositories = array_keys(self::getSupportedRepositories());
+        $repositories = array_keys(self::getSupportedRepositories($exclude_meta));
         sort($repositories);
 
         return $repositories;
@@ -154,11 +159,13 @@ public static function getRepositories()
      * The array has repo folder names as keys and Display names as value:
      * ex: ['firefox_ios' => 'Firefox for iOS', 'mozilla_org' => 'mozilla.org']
      *
+     * @param boolean $exclude_meta Exclude meta project
+     *
      * @return array list of local repositories and their Display names
      */
-    public static function getRepositoriesNames()
+    public static function getRepositoriesNames($exclude_meta = false)
     {
-        return self::getSupportedRepositories();
+        return self::getSupportedRepositories($exclude_meta);
     }
 
     /**
@@ -214,6 +221,23 @@ public static function getRepositoryLocales($repository, $ignored = [])
         return $supported_locales;
     }
 
+    /**
+     * Get the list of all locales available by looking at the meta project
+     *
+     * @return array A sorted list of all supported locales
+     */
+    public static function getAllLocales()
+    {
+        $file_name = APP_SOURCES . 'all_projects.txt';
+        $supported_locales = [];
+        if (file_exists($file_name)) {
+            $supported_locales = file($file_name,  FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
+        }
+        sort($supported_locales);
+
+        return $supported_locales;
+    }
+
     /**
      * Get the list of repositories available for a locale
      *
@@ -224,7 +248,7 @@ public static function getRepositoryLocales($repository, $ignored = [])
     public static function getLocaleRepositories($locale)
     {
         $matches = [];
-        foreach (self::getRepositories() as $repository) {
+        foreach (self::getRepositories(true) as $repository) {
             if (in_array($locale, self::getRepositoryLocales($repository))) {
                 $matches[] = $repository;
             }
@@ -238,7 +262,7 @@ public static function getLocaleRepositories($locale)
     /**
      * Return the reference locale for a repository
      *
-     * @param string $repository Name of the folder for the repository
+     * @param string $repository Name of the repository
      *
      * @return string Name of the reference locale
      */
@@ -249,10 +273,37 @@ public static function getReferenceLocale($repository)
             : 'en-US';
     }
 
+    /**
+     * Return true if it's a meta repository
+     *
+     * @param string $repository Name of the repository
+     *
+     * @return boolean True if the repository is set as meta
+     */
+    public static function isMetaRepository($repository)
+    {
+        return isset(self::$repos_info[$repository]['meta'])
+            ? self::$repos_info[$repository]['meta']
+            : false;
+    }
+
+    /**
+     * Return true if the locale is the reference locale for a repository
+     *
+     * @param string $locale     Locale code
+     * @param string $repository Name of the repository
+     *
+     * @return boolean True is it's the reference locale
+     */
+    public static function isReferenceLocale($locale, $repository)
+    {
+        return self::getReferenceLocale($repository) == $locale;
+    }
+
     /**
      * Check if the specified repository is supported
      *
-     * @param string $repository Name of the folder for the repository
+     * @param string $repository Name of the repository
      *
      * @return boolean True if supported repository, False if unknown
      */

app/classes/Transvision/Search.php

@@ -98,7 +98,7 @@ class Search
      */
     protected $form_search_options = [
         'case_sensitive', 'entire_string', 'repo',
-        'search_type', 't2t', 'each_word', 'entire_words',
+        'search_type', 'each_word', 'entire_words',
     ];
 
     /**
@@ -332,13 +332,23 @@ public function getRegexSearchTerms()
     /**
      * Grep data in regex
      *
-     * @param array $source_strings The array of strings to be filtered
+     * @param array   $source_strings The array of strings to be filtered
+     * @param boolean $flat_source    If true, the source source is a flat array
      *
      * @return array Return an array of filtered strings
      */
-    public function grep($source_strings)
+    public function grep($source_strings, $flat = true)
     {
-        return preg_grep($this->getRegex(), $source_strings);
+        if ($flat) {
+            return preg_grep($this->getRegex(), $source_strings);
+        } else {
+            $results = [];
+            foreach ($source_strings as $repo => $strings) {
+                $results[$repo] = preg_grep($this->getRegex(), $strings);
+            }
+
+            return $results;
+        }
     }
 
     /**