Split user language (UI translation) from locale (formatted values)

config/services.yaml

@@ -1,8 +1,7 @@
 parameters:
     locale: en
-    # the original list of all locales is the next line, it can be used to regenerate the locale list in case new locales will be added
-    #app_locales: ar|cs|da|de|el|en|eo|es|eu|fa|fi|fo|fr|he|hr|hu|it|ja|ko|nb_NO|nl|pl|pt|ro|ru|sk|sv|tr|uk|vi|zh_CN|zh_Hant
-    app_locales: ar|cs|da|de|el|en|eo|es|eu|fa|fi|fo|fr|he|hr|hu|it|ja|ko|nb_NO|nl|pl|pt|pt_BR|ro|ru|sk|sv|tr|uk|vi|zh_CN|zh_Hant|cs_CZ|da_DK|da_GL|de_AT|de_BE|de_CH|de_DE|de_IT|de_LI|de_LU|el_CY|el_GR|en_AE|en_AG|en_AI|en_AS|en_AT|en_AU|en_BB|en_BE|en_BI|en_BM|en_BS|en_BW|en_BZ|en_CA|en_CC|en_CH|en_CK|en_CM|en_CX|en_CY|en_DE|en_DK|en_DM|en_ER|en_FI|en_FJ|en_FK|en_FM|en_GB|en_GD|en_GG|en_GH|en_GI|en_GM|en_GU|en_GY|en_HK|en_IE|en_IL|en_IM|en_IN|en_IO|en_JE|en_JM|en_KE|en_KI|en_KN|en_KY|en_LC|en_LR|en_LS|en_MG|en_MH|en_MO|en_MP|en_MS|en_MT|en_MU|en_MV|en_MW|en_MY|en_NA|en_NF|en_NG|en_NL|en_NR|en_NU|en_NZ|en_PG|en_PH|en_PK|en_PN|en_PR|en_PW|en_RW|en_SB|en_SC|en_SD|en_SE|en_SG|en_SH|en_SI|en_SL|en_SS|en_SX|en_SZ|en_TC|en_TK|en_TO|en_TT|en_TV|en_TZ|en_UG|en_UM|en_US|en_VC|en_VG|en_VI|en_VU|en_WS|en_ZA|en_ZM|en_ZW|es_AR|es_BO|es_BR|es_BZ|es_CL|es_CO|es_CR|es_CU|es_DO|es_EC|es_ES|es_GQ|es_GT|es_HN|es_MX|es_NI|es_PA|es_PE|es_PH|es_PR|es_PY|es_SV|es_US|es_UY|es_VE|eu_ES|fa_AF|fa_IR|fi_FI|fo_DK|fo_FO|fr_BE|fr_BF|fr_BI|fr_BJ|fr_BL|fr_CA|fr_CD|fr_CF|fr_CG|fr_CH|fr_CI|fr_CM|fr_DJ|fr_DZ|fr_FR|fr_GA|fr_GF|fr_GN|fr_GP|fr_GQ|fr_HT|fr_KM|fr_LU|fr_MA|fr_MC|fr_MF|fr_MG|fr_ML|fr_MQ|fr_MR|fr_MU|fr_NC|fr_NE|fr_PF|fr_PM|fr_RE|fr_RW|fr_SC|fr_SN|fr_SY|fr_TD|fr_TG|fr_TN|fr_VU|fr_WF|fr_YT|he_IL|hr_BA|hr_HR|hu_HU|it_CH|it_IT|it_SM|it_VA|ja_JP|ko_KP|ko_KR|nl_AW|nl_BE|nl_BQ|nl_CW|nl_NL|nl_SR|nl_SX|pl_PL|pt_AO|pt_CH|pt_CV|pt_GQ|pt_GW|pt_LU|pt_MO|pt_MZ|pt_PT|pt_ST|pt_TL|ro_MD|ro_RO|ru_BY|ru_KG|ru_KZ|ru_MD|ru_RU|ru_UA|sk_SK|sv_AX|sv_FI|sv_SE|tr_CY|tr_TR|vi_VN
+    # can be regenerated with "bin/console kimai:reset:locales"
+    app_locales: ar|cs|da|de|de_CH|el|en|eo|es|eu|fa|fi|fo|fr|he|hr|hu|id|it|ja|ko|nb_NO|nl|pa|pl|pt|pt_BR|ro|ru|sk|sl|sv|tr|uk|vi|zh_CN|zh_Hant|cs_CZ|da_DK|da_GL|de_AT|de_BE|de_CH|de_DE|de_IT|de_LI|de_LU|el_CY|el_GR|en_AE|en_AG|en_AI|en_AS|en_AT|en_AU|en_BB|en_BE|en_BI|en_BM|en_BS|en_BW|en_BZ|en_CA|en_CC|en_CH|en_CK|en_CM|en_CX|en_CY|en_DE|en_DG|en_DK|en_DM|en_ER|en_FI|en_FJ|en_FK|en_FM|en_GB|en_GD|en_GG|en_GH|en_GI|en_GM|en_GU|en_GY|en_HK|en_ID|en_IE|en_IL|en_IM|en_IN|en_IO|en_JE|en_JM|en_KE|en_KI|en_KN|en_KY|en_LC|en_LR|en_LS|en_MG|en_MH|en_MO|en_MP|en_MS|en_MT|en_MU|en_MV|en_MW|en_MY|en_NA|en_NF|en_NG|en_NH|en_NL|en_NR|en_NU|en_NZ|en_PG|en_PH|en_PK|en_PN|en_PR|en_PW|en_RH|en_RW|en_SB|en_SC|en_SD|en_SE|en_SG|en_SH|en_SI|en_SL|en_SS|en_SX|en_SZ|en_TC|en_TK|en_TO|en_TT|en_TV|en_TZ|en_UG|en_UM|en_US|en_VC|en_VG|en_VI|en_VU|en_WS|en_ZA|en_ZM|en_ZW|es_AR|es_BO|es_BR|es_BZ|es_CL|es_CO|es_CR|es_CU|es_DO|es_EA|es_EC|es_ES|es_GQ|es_GT|es_HN|es_IC|es_MX|es_NI|es_PA|es_PE|es_PH|es_PR|es_PY|es_SV|es_US|es_UY|es_VE|eu_ES|fa_AF|fa_IR|fi_FI|fo_DK|fo_FO|fr_BE|fr_BF|fr_BI|fr_BJ|fr_BL|fr_CA|fr_CD|fr_CF|fr_CG|fr_CH|fr_CI|fr_CM|fr_DJ|fr_DZ|fr_FR|fr_GA|fr_GF|fr_GN|fr_GP|fr_GQ|fr_HT|fr_KM|fr_LU|fr_MA|fr_MC|fr_MF|fr_MG|fr_ML|fr_MQ|fr_MR|fr_MU|fr_NC|fr_NE|fr_PF|fr_PM|fr_RE|fr_RW|fr_SC|fr_SN|fr_SY|fr_TD|fr_TG|fr_TN|fr_VU|fr_WF|fr_YT|he_IL|hr_BA|hr_HR|hu_HU|it_CH|it_IT|it_SM|it_VA|ja_JP|ko_CN|ko_KP|ko_KR|nl_AW|nl_BE|nl_BQ|nl_CW|nl_NL|nl_SR|nl_SX|pl_PL|pt_AO|pt_BR|pt_CH|pt_CV|pt_GQ|pt_GW|pt_LU|pt_MO|pt_MZ|pt_PT|pt_ST|pt_TL|ro_MD|ro_RO|ru_BY|ru_KG|ru_KZ|ru_MD|ru_RU|ru_UA|sk_SK|sv_AX|sv_FI|sv_SE|tr_CY|tr_TR|uk_UA|vi_VN
 
 services:
     # default configuration for services in *this* file

phpstan.neon

@@ -492,11 +492,6 @@ parameters:
             count: 1
             path: src/Command/RegenerateLocalesCommand.php
 
-        -
-            message: "#^Property App\\\\Command\\\\RegenerateLocalesCommand\\:\\:\\$rtlLocales type has no value type specified in iterable type array\\.$#"
-            count: 1
-            path: src/Command/RegenerateLocalesCommand.php
-
         -
             message: "#^Cannot call method find\\(\\) on Symfony\\\\Component\\\\Console\\\\Application\\|null\\.$#"
             count: 4
@@ -652,26 +647,6 @@ parameters:
             count: 1
             path: src/Configuration/LdapConfiguration.php
 
-        -
-            message: "#^Method App\\\\Configuration\\\\LocaleService\\:\\:__construct\\(\\) has parameter \\$languageSettings with no value type specified in iterable type array\\.$#"
-            count: 1
-            path: src/Configuration/LocaleService.php
-
-        -
-            message: "#^Method App\\\\Configuration\\\\LocaleService\\:\\:getDateFormat\\(\\) should return string but returns bool\\|string\\.$#"
-            count: 1
-            path: src/Configuration/LocaleService.php
-
-        -
-            message: "#^Method App\\\\Configuration\\\\LocaleService\\:\\:getTimeFormat\\(\\) should return string but returns bool\\|string\\.$#"
-            count: 1
-            path: src/Configuration/LocaleService.php
-
-        -
-            message: "#^Method App\\\\Configuration\\\\LocaleService\\:\\:isRightToLeft\\(\\) should return bool but returns bool\\|string\\.$#"
-            count: 1
-            path: src/Configuration/LocaleService.php
-
         -
             message: "#^Method App\\\\Configuration\\\\SamlConfiguration\\:\\:getAttributeMapping\\(\\) return type has no value type specified in iterable type array\\.$#"
             count: 1
@@ -1607,11 +1582,6 @@ parameters:
             count: 1
             path: src/Entity/User.php
 
-        -
-            message: "#^Method App\\\\Entity\\\\User\\:\\:getLocale\\(\\) should return string but returns bool\\|float\\|int\\|string\\|null\\.$#"
-            count: 1
-            path: src/Entity/User.php
-
         -
             message: "#^Method App\\\\Entity\\\\User\\:\\:getMemberships\\(\\) return type with generic interface Doctrine\\\\Common\\\\Collections\\\\Collection does not specify its types\\: TKey, T$#"
             count: 1
@@ -5407,11 +5377,6 @@ parameters:
             count: 1
             path: src/Utils/FileHelper.php
 
-        -
-            message: "#^Property App\\\\Utils\\\\FormFormatConverter\\:\\:\\$formatConvertRules type has no value type specified in iterable type array\\.$#"
-            count: 1
-            path: src/Utils/FormFormatConverter.php
-
         -
             message: "#^Property App\\\\Utils\\\\JavascriptFormatConverter\\:\\:\\$formatConvertRules type has no value type specified in iterable type array\\.$#"
             count: 1

src/API/Serializer/ValidationFailedExceptionErrorHandler.php

@@ -83,7 +83,7 @@ private function getErrorMessage(ConstraintViolationInterface $error): string
         $user = $this->security->getUser();
 
         if ($user !== null) {
-            $locale = $user->getLocale();
+            $locale = $user->getLanguage();
         }
 
         if (null !== $error->getPlural()) {

src/Command/RegenerateLocalesCommand.php

@@ -29,15 +29,22 @@
 #[AsCommand(name: 'kimai:reset:locales')]
 final class RegenerateLocalesCommand extends Command
 {
-    private string $defaultDate = 'dd.MM.y';
-    private string $defaultTime = 'HH:mm';
-    private array $rtlLocales = [
-        'ar' => true,
-        'fa' => true,
-        'he' => true,
-    ];
-
-    public function __construct(private LocaleService $localeService, private string $projectDirectory, private string $kernelEnvironment)
+    /**
+     * @var string[]
+     */
+    private array $rtlLocales = ['ar', 'fa', 'he'];
+    /**
+     * new locales were added here, to shrink the list a little bit
+     * this can be removed in the future, if there will ever be the need for it
+     *
+     * @var string[]
+     */
+    private array $noRegionCode = ['ar', 'id', 'pa', 'sl'];
+
+    public function __construct(
+        private readonly string $projectDirectory,
+        private readonly string $kernelEnvironment
+    )
     {
         parent::__construct();
     }
@@ -55,44 +62,58 @@ protected function configure(): void
     protected function execute(InputInterface $input, OutputInterface $output): int
     {
         $io = new SymfonyStyle($input, $output);
-        $locales = $this->localeService->getAllLocales();
 
-        // detect all registered locales and allow to choose them as well, so people get to
-        // choose the language for translation with the correct format of their location
-        /*
+        // find all available locales from the translation filenames
+        $translationFilenames = glob($this->projectDirectory . DIRECTORY_SEPARATOR . 'translations/*.xlf');
+        if ($translationFilenames === false) {
+            $io->error('Failed reading translation files');
+
+            return Command::FAILURE;
+        }
+        $firstLevelLocales = [];
+        foreach ($translationFilenames as $file) {
+            $firstLevelLocales[] = explode('.', basename($file))[1];
+        }
+        $firstLevelLocales = array_unique($firstLevelLocales);
+        $io->title('Locales found from translation files');
+        $io->writeln(implode('|', $firstLevelLocales));
+
         $secondLevel = [];
-        foreach (Locales::getLocales() as $locale) {
-            if (substr_count($locale, '_') === 1) {
-                $baseLocale = substr($locale, 0, strpos($locale, '_'));
-                if (in_array($baseLocale, $locales)) {
-                    $subLocale = substr($locale, strpos($locale, '_') + 1);
-                    if (!is_numeric($subLocale)) {
-                        $secondLevel[] = $locale;
+        foreach (Locales::getLocales() as $localeCode) {
+            $locale = explode('_', $localeCode);
+            if (\count($locale) === 2 && !\in_array($locale[0], $this->noRegionCode, true)) {
+                $baseLocale = $locale[0];
+                if (\in_array($baseLocale, $firstLevelLocales)) {
+                    $regionCode = $locale[1];
+                    if (!is_numeric($regionCode)) {
+                        $secondLevel[] = $localeCode;
                     }
                 }
             }
         }
-        $locales = array_merge($locales, $secondLevel);
-        */
+
+        sort($firstLevelLocales);
+        sort($secondLevel);
+
+        // keep the locales that have translation filesat the begin
+        // the config is than easier to read and the locales will be sorted in the UI anyway
+        $locales = array_merge($firstLevelLocales, $secondLevel);
 
         $appLocales = [];
-        $defaults = [
-            'date' => $this->defaultDate,
-            'time' => $this->defaultTime,
-            'rtl' => false,
-        ];
 
         // make sure all allowed locales are registered
         foreach ($locales as $locale) {
             if (!Locales::exists($locale)) {
                 continue;
             }
 
-            $appLocales[$locale] = $defaults;
+            $appLocales[$locale] = LocaleService::DEFAULT_SETTINGS;
         }
 
         // make sure all keys are registered for every locale
         foreach ($appLocales as $locale => $settings) {
+            $settings['translation'] = \in_array($locale, $firstLevelLocales, true);
+
             // these are completely new since v2
             // calculate everything with IntlFormatter
             $shortDate = new \IntlDateFormatter($locale, \IntlDateFormatter::SHORT, \IntlDateFormatter::NONE);
@@ -115,14 +136,50 @@ protected function execute(InputInterface $input, OutputInterface $output): int
                 $rtlLocale = substr($rtlLocale, 0, strpos($rtlLocale, '_'));
             }
 
-            if (\array_key_exists($rtlLocale, $this->rtlLocales)) {
-                $settings['rtl'] = $this->rtlLocales[$rtlLocale];
-            }
+            $settings['rtl'] = \in_array($rtlLocale, $this->rtlLocales, true);
 
             // pre-fill all formats with the default locale settings
             $appLocales[$locale] = $settings;
         }
 
+        $removableDuplicates = [];
+        foreach ($appLocales as $locale => $setting) {
+            $localeParts = explode('_', $locale);
+            if (\count($localeParts) === 1) {
+                continue;
+            }
+            // e.g. norwegian just exists with region code
+            if (!\array_key_exists($localeParts[0], $appLocales)) {
+                continue;
+            }
+            $baseLocaleSettings = $appLocales[$localeParts[0]];
+            if ($baseLocaleSettings['time'] !== $setting['time']) {
+                continue;
+            }
+            if ($baseLocaleSettings['date'] !== $setting['date']) {
+                continue;
+            }
+            if ($setting['translation'] === true) {
+                continue;
+            }
+            if ($baseLocaleSettings['rtl'] !== $setting['rtl']) {
+                continue;
+            }
+            $removableDuplicates[] = $locale;
+        }
+
+        $io->title('Redundant locales that will be skipped');
+        $io->writeln(implode('|', $removableDuplicates));
+
+        foreach ($removableDuplicates as $duplicate) {
+            unset($appLocales[$duplicate]);
+        }
+
+        // in the future this list should be reduced to the list of available translations, but for a long time users
+        // could choose from the entire list of all locales, so we likely have to keep that forever ...
+        $io->title('List of app_locales for services.yaml');
+        $io->writeln(implode('|', $locales));
+
         ksort($appLocales);
 
         $filename = 'config/locales.php';

src/Command/UserLoginLinkCommand.php

@@ -70,7 +70,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
         }
 
         $request = new Request();
-        $request->setLocale($user->getLocale());
+        $request->setLocale($user->getLanguage());
         $this->requestStack->push($request);
 
         $loginLinkDetails = $this->loginLink->createLoginLink($user, $request);

src/Configuration/LocaleService.php

@@ -9,9 +9,21 @@
 
 namespace App\Configuration;
 
+use App\Entity\User;
+
 final class LocaleService
 {
-    public function __construct(private array $languageSettings)
+    public const DEFAULT_SETTINGS = [
+        'date' => 'dd.MM.y',
+        'time' => 'HH:mm',
+        'rtl' => false,
+        'translation' => false,
+    ];
+
+    /**
+     * @param array<string, array{'date': string, 'time': string, 'translation': bool}> $languageSettings
+     */
+    public function __construct(private readonly array $languageSettings)
     {
     }
 
@@ -25,6 +37,18 @@ public function getAllLocales(): array
         return array_keys($this->languageSettings);
     }
 
+    /**
+     * Returns an array with all language codes that have translations.
+     *
+     * @return string[]
+     */
+    public function getTranslatedLocales(): array
+    {
+        return array_keys(array_filter($this->languageSettings, function (array $setting) {
+            return $setting['translation'];
+        }));
+    }
+
     public function isKnownLocale(string $language): bool
     {
         return \in_array($language, $this->getAllLocales());
@@ -38,7 +62,7 @@ public function isKnownLocale(string $language): bool
      */
     public function getDateFormat(string $locale): string
     {
-        return $this->getConfig('date', $locale);
+        return (string) $this->getConfig('date', $locale);
     }
 
     /**
@@ -49,7 +73,7 @@ public function getDateFormat(string $locale): string
      */
     public function getTimeFormat(string $locale): string
     {
-        return $this->getConfig('time', $locale);
+        return (string) $this->getConfig('time', $locale);
     }
 
     /**
@@ -76,7 +100,34 @@ public function getDurationFormat(string $locale): string
 
     public function isRightToLeft(string $locale): bool
     {
-        return $this->getConfig('rtl', $locale);
+        return (bool) $this->getConfig('rtl', $locale);
+    }
+
+    public function isTranslated(string $locale): bool
+    {
+        return (bool) $this->getConfig('translation', $locale);
+    }
+
+    public function getNearestTranslationLocale(string $locale): string
+    {
+        if (!$this->isKnownLocale($locale)) {
+            $parts = explode('_', $locale);
+            if (\count($parts) !== 2 || \strlen($parts[0]) !== 2 || !$this->isKnownLocale($parts[0])) {
+                return User::DEFAULT_LANGUAGE;
+            }
+            $locale = $parts[0];
+        }
+
+        if (!$this->isTranslated($locale)) {
+            $base = explode('_', $locale)[0];
+            if (!$this->isTranslated($base)) {
+                return User::DEFAULT_LANGUAGE;
+            }
+
+            return $base;
+        }
+
+        return $locale;
     }
 
     public function is24Hour(string $locale): bool

src/Controller/SystemConfigurationController.php

@@ -22,13 +22,13 @@
 use App\Form\Type\DatePickerType;
 use App\Form\Type\DateTimeTextType;
 use App\Form\Type\DayTimeType;
-use App\Form\Type\LanguageType;
 use App\Form\Type\MinuteIncrementType;
 use App\Form\Type\ProjectTypePatternType;
 use App\Form\Type\RoundingModeType;
 use App\Form\Type\SkinType;
 use App\Form\Type\TimezoneType;
 use App\Form\Type\TrackingModeType;
+use App\Form\Type\UserLanguageType;
 use App\Form\Type\WeekDaysType;
 use App\Form\Type\YesNoType;
 use App\Timesheet\LockdownService;
@@ -504,7 +504,7 @@ private function getConfigurationTypes(): array
                         ->setOptions(['help' => 'default_value_new']),
                     (new Configuration('defaults.user.language'))
                         ->setLabel('language')
-                        ->setType(LanguageType::class)
+                        ->setType(UserLanguageType::class)
                         ->setOptions(['help' => 'default_value_new']),
                     (new Configuration('defaults.user.theme'))
                         ->setLabel('skin')