Use Locale IDs for tl_page.language (see #2305)

Description
-----------

| Q                | A
| -----------------| ---
| Fixed issues     | Fixes contao/contao#1957

Since Contao 4.10, the URL prefix is no longer tied to the page language. With this PR, the page language can be any valid Locale ID. A Locale ID can include language, script, country and additional information (e.g. `de_Latn_CH@currency=EUR`),  see https://github.com/contao/core-bundle/issues/233.

Locale ID and ICU information can later be used a lot more things like rewriting the number formatting to use the ICU information (decimal point according to language & country).

Commits
-------

c45065b5 Use Locale ID for tl_page.language
832f980f Migrate the existing page languages
4372b7e3 Format locale instead of string replace
7fa7cf79 Support old locale style in legacy routing
f5b0c0c5 Added support for full locale routing
af357553 Do not migrate the page language
4b5c98a0 Updated hint for page language field
6d303ea6 Use a listener for the page language callback
40bfc173 Use method to calculate locale priority
0c22b1bb CS
bc4abe59 Fixed tests
95293af0 Feedback adjustments
3a853b8f Migrate the page languages
de7b8d7c Always convert page language and make sure it starts with two letters
c2ca955d Only support correctly formatted language folders
e149b37a Fix user and member language fields
300d40f2 Fixed rebase issues
edfff9ad CS
bfb0cf68 Merge branch '4.x' into feature/locale
3ad4a000 CS

src/DependencyInjection/Configuration.php

@@ -12,6 +12,7 @@
 
 namespace Contao\CoreBundle\DependencyInjection;
 
+use Contao\CoreBundle\Util\LocaleUtil;
 use Contao\Image\ResizeConfiguration;
 use Imagine\Image\ImageInterface;
 use Symfony\Component\Config\Definition\Builder\NodeDefinition;
@@ -514,10 +515,16 @@ private function getLocales(): array
         $languages = [$this->defaultLocale];
 
         /** @var array<SplFileInfo> $finder */
-        $finder = Finder::create()->directories()->depth(0)->name('/^[a-z]{2}(_[A-Z]{2})?$/')->in($dirs);
+        $finder = Finder::create()->directories()->depth(0)->name('/^[a-z]{2,}/')->in($dirs);
 
         foreach ($finder as $file) {
-            $languages[] = $file->getFilename();
+            $locale = $file->getFilename();
+
+            if (LocaleUtil::canonicalize($locale) !== $locale) {
+                continue;
+            }
+
+            $languages[] = $locale;
         }
 
         return array_values(array_unique($languages));

src/EventListener/BackendLocaleListener.php

@@ -13,6 +13,7 @@
 namespace Contao\CoreBundle\EventListener;
 
 use Contao\BackendUser;
+use Contao\CoreBundle\Util\LocaleUtil;
 use Symfony\Component\HttpKernel\Event\RequestEvent;
 use Symfony\Component\Security\Core\Security;
 use Symfony\Contracts\Translation\LocaleAwareInterface;
@@ -55,6 +56,6 @@ public function __invoke(RequestEvent $event): void
         $this->translator->setLocale($user->language);
 
         // Deprecated since Contao 4.0, to be removed in Contao 5.0
-        $GLOBALS['TL_LANGUAGE'] = str_replace('_', '-', $user->language);
+        $GLOBALS['TL_LANGUAGE'] = LocaleUtil::formatAsLanguageTag($user->language);
     }
 }

src/EventListener/DataContainer/LegacyRoutingListener.php

@@ -14,6 +14,7 @@
 
 use Contao\CoreBundle\Framework\ContaoFramework;
 use Contao\CoreBundle\ServiceAnnotation\Callback;
+use Contao\CoreBundle\Util\LocaleUtil;
 use Contao\DataContainer;
 use Contao\Image;
 use Contao\StringUtil;
@@ -83,7 +84,7 @@ public function disableRoutingFields(): void
      */
     public function overrideUrlPrefix($value, DataContainer $dc): ?string
     {
-        return $this->prependLocale ? $dc->activeRecord->language : '';
+        return $this->prependLocale ? LocaleUtil::formatAsLanguageTag($dc->activeRecord->language) : '';
     }
 
     /**

src/EventListener/LocaleSubscriber.php

@@ -13,6 +13,7 @@
 namespace Contao\CoreBundle\EventListener;
 
 use Contao\CoreBundle\Routing\ScopeMatcher;
+use Contao\CoreBundle\Util\LocaleUtil;
 use Symfony\Component\EventDispatcher\EventSubscriberInterface;
 use Symfony\Component\HttpFoundation\Request;
 use Symfony\Component\HttpKernel\Event\RequestEvent;
@@ -85,25 +86,9 @@ public static function getSubscribedEvents(): array
     private function getLocale(Request $request): string
     {
         if (null !== $request->attributes->get('_locale')) {
-            return $this->formatLocaleId($request->attributes->get('_locale'));
+            return LocaleUtil::formatAsLocale($request->attributes->get('_locale'));
         }
 
         return $request->getPreferredLanguage($this->availableLocales);
     }
-
-    private function formatLocaleId(string $locale): string
-    {
-        if (!preg_match('/^[a-z]{2}([_-][a-z]{2})?$/i', $locale)) {
-            throw new \InvalidArgumentException(sprintf('"%s" is not a supported locale.', $locale));
-        }
-
-        $values = preg_split('/[_-]/', $locale);
-        $locale = strtolower($values[0]);
-
-        if (isset($values[1])) {
-            $locale .= '_'.strtoupper($values[1]);
-        }
-
-        return $locale;
-    }
 }

src/Framework/ContaoFramework.php

@@ -19,6 +19,7 @@
 use Contao\CoreBundle\Routing\ScopeMatcher;
 use Contao\CoreBundle\Security\Authentication\Token\TokenChecker;
 use Contao\CoreBundle\Session\LazySessionAccess;
+use Contao\CoreBundle\Util\LocaleUtil;
 use Contao\Environment;
 use Contao\Input;
 use Contao\InsertTags;
@@ -351,7 +352,7 @@ private function setDefaultLanguage(): void
         $language = 'en';
 
         if (null !== $this->request) {
-            $language = str_replace('_', '-', $this->request->getLocale());
+            $language = LocaleUtil::formatAsLanguageTag($this->request->getLocale());
         }
 
         // Deprecated since Contao 4.0, to be removed in Contao 5.0

src/Image/Studio/FigureBuilder.php

@@ -15,6 +15,7 @@
 use Contao\CoreBundle\Event\FileMetadataEvent;
 use Contao\CoreBundle\Exception\InvalidResourceException;
 use Contao\CoreBundle\File\Metadata;
+use Contao\CoreBundle\Util\LocaleUtil;
 use Contao\FilesModel;
 use Contao\Image\ImageInterface;
 use Contao\Image\PictureConfiguration;
@@ -756,10 +757,10 @@ private function getFallbackLocaleList(): array
             return [];
         }
 
-        $locales = [str_replace('-', '_', $page->language)];
+        $locales = [LocaleUtil::formatAsLocale($page->language)];
 
         if (null !== $page->rootFallbackLanguage) {
-            $locales[] = str_replace('-', '_', $page->rootFallbackLanguage);
+            $locales[] = LocaleUtil::formatAsLocale($page->rootFallbackLanguage);
         }
 
         return array_unique(array_filter($locales));

src/Migration/Version412/PageLanguageMigration.php

@@ -0,0 +1,82 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of Contao.
+ *
+ * (c) Leo Feyer
+ *
+ * @license LGPL-3.0-or-later
+ */
+
+namespace Contao\CoreBundle\Migration\Version412;
+
+use Contao\CoreBundle\Migration\AbstractMigration;
+use Contao\CoreBundle\Migration\MigrationResult;
+use Contao\CoreBundle\Util\LocaleUtil;
+use Doctrine\DBAL\Connection;
+
+/**
+ * @internal
+ */
+class PageLanguageMigration extends AbstractMigration
+{
+    /**
+     * @var Connection
+     */
+    private $connection;
+
+    public function __construct(Connection $connection)
+    {
+        $this->connection = $connection;
+    }
+
+    public function shouldRun(): bool
+    {
+        $schemaManager = $this->connection->getSchemaManager();
+
+        if (!$schemaManager->tablesExist(['tl_page'])) {
+            return false;
+        }
+
+        $pageColumns = $schemaManager->listTableColumns('tl_page');
+
+        if (!isset($pageColumns['language'])) {
+            return false;
+        }
+
+        $count = $this->connection->fetchOne("
+            SELECT
+                COUNT(*)
+            FROM
+                tl_page
+            WHERE
+                type='root' AND SUBSTRING(language, 3, 1) = '-'
+        ");
+
+        return $count > 0;
+    }
+
+    public function run(): MigrationResult
+    {
+        $pages = $this->connection->fetchAllAssociative("
+            SELECT
+                id, language
+            FROM
+                tl_page
+            WHERE
+                type='root' AND SUBSTRING(language, 3, 1) = '-'
+        ");
+
+        foreach ($pages as $page) {
+            $this->connection->update(
+                'tl_page',
+                ['language' => LocaleUtil::canonicalize($page['language'])],
+                ['id' => $page['id']]
+            );
+        }
+
+        return $this->createResult(true);
+    }
+}

src/Resources/config/migrations.yml

@@ -83,3 +83,7 @@ services:
     Contao\CoreBundle\Migration\Version412\AllowedExcludedFieldsMigration:
         arguments:
             - '@database_connection'
+
+    Contao\CoreBundle\Migration\Version412\PageLanguageMigration:
+        arguments:
+            - '@database_connection'

src/Resources/contao/classes/Backend.php

@@ -13,6 +13,7 @@
 use Contao\CoreBundle\Exception\AccessDeniedException;
 use Contao\CoreBundle\Exception\ResponseException;
 use Contao\CoreBundle\Picker\PickerInterface;
+use Contao\CoreBundle\Util\LocaleUtil;
 use Contao\Database\Result;
 use Symfony\Component\Filesystem\Exception\IOException;
 use Symfony\Component\Filesystem\Filesystem;
@@ -90,14 +91,13 @@ public static function getThemes()
 	 */
 	public static function getTinyMceLanguage()
 	{
-		$lang = $GLOBALS['TL_LANGUAGE'];
+		$lang = LocaleUtil::formatAsLocale((string) $GLOBALS['TL_LANGUAGE']);
 
 		if (!$lang)
 		{
 			return 'en';
 		}
 
-		$lang = str_replace('-', '_', $lang);
 		$projectDir = System::getContainer()->getParameter('kernel.project_dir');
 
 		// The translation exists
@@ -778,7 +778,7 @@ public static function addFileMetaInformationToRequest($strUuid, $strPtable, $in
 		$objPage->loadDetails();
 
 		// Convert the language to a locale (see #5678)
-		$strLanguage = str_replace('-', '_', $objPage->rootLanguage);
+		$strLanguage = LocaleUtil::formatAsLocale($objPage->rootLanguage);
 
 		if (isset($arrMeta[$strLanguage]))
 		{

src/Resources/contao/classes/Frontend.php

@@ -13,6 +13,7 @@
 use Contao\CoreBundle\Exception\LegacyRoutingException;
 use Contao\CoreBundle\Exception\NoRootPageFoundException;
 use Contao\CoreBundle\Search\Document;
+use Contao\CoreBundle\Util\LocaleUtil;
 use Symfony\Cmf\Component\Routing\RouteObjectInterface;
 use Symfony\Component\HttpFoundation\Request;
 use Symfony\Component\HttpFoundation\Response;
@@ -380,7 +381,7 @@ public static function getRootPageFromUrl()
 		{
 			if ($blnAddLanguageToUrl)
 			{
-				$arrParams = array('_locale' => $objRootPage->language);
+				$arrParams = array('_locale' => LocaleUtil::formatAsLocale($objRootPage->language));
 
 				$strUrl = System::getContainer()->get('router')->generate('contao_index', $arrParams);
 				$strUrl = substr($strUrl, \strlen(Environment::get('path')) + 1);
@@ -558,7 +559,7 @@ public static function getMetaData($strData, $strLanguage)
 		$arrData = StringUtil::deserialize($strData);
 
 		// Convert the language to a locale (see #5678)
-		$strLanguage = str_replace('-', '_', $strLanguage);
+		$strLanguage = LocaleUtil::formatAsLocale($strLanguage);
 
 		if (!\is_array($arrData) || !isset($arrData[$strLanguage]))
 		{

src/Resources/contao/controllers/BackendIndex.php

@@ -111,7 +111,6 @@ public function run()
 		$objTemplate->host = Backend::getDecodedHostname();
 		$objTemplate->charset = System::getContainer()->getParameter('kernel.charset');
 		$objTemplate->userLanguage = $GLOBALS['TL_LANG']['tl_user']['language'][0];
-		$objTemplate->curLanguage = Input::post('language') ?: str_replace('-', '_', $GLOBALS['TL_LANGUAGE']);
 		$objTemplate->curUsername = Input::post('username') ?: '';
 		$objTemplate->loginButton = StringUtil::specialchars($GLOBALS['TL_LANG']['MSC']['continue']);
 		$objTemplate->username = $GLOBALS['TL_LANG']['tl_user']['username'][0];

src/Resources/contao/controllers/FrontendIndex.php

@@ -14,6 +14,7 @@
 use Contao\CoreBundle\Exception\InsufficientAuthenticationException;
 use Contao\CoreBundle\Exception\PageNotFoundException;
 use Contao\CoreBundle\Security\ContaoCorePermissions;
+use Contao\CoreBundle\Util\LocaleUtil;
 use Contao\Model\Collection;
 use Symfony\Component\HttpFoundation\Response;
 
@@ -247,8 +248,8 @@ public function renderPage($pageModel)
 			throw new PageNotFoundException('Page not found: ' . Environment::get('uri'));
 		}
 
-		// Check wether the language matches the root page language
-		if (isset($_GET['language']) && $objPage->urlPrefix && Input::get('language') != $objPage->rootLanguage)
+		// Check whether the language matches the root page language
+		if (isset($_GET['language']) && $objPage->urlPrefix && Input::get('language') != LocaleUtil::formatAsLanguageTag($objPage->rootLanguage))
 		{
 			throw new PageNotFoundException('Page not found: ' . Environment::get('uri'));
 		}

src/Resources/contao/dca/tl_member.php

@@ -271,12 +271,12 @@
 			'exclude'                 => true,
 			'filter'                  => true,
 			'inputType'               => 'select',
-			'eval'                    => array('includeBlankOption'=>true, 'chosen'=>true, 'rgxp'=>'locale', 'feEditable'=>true, 'feViewable'=>true, 'feGroup'=>'personal', 'tl_class'=>'w50'),
+			'eval'                    => array('includeBlankOption'=>true, 'chosen'=>true, 'feEditable'=>true, 'feViewable'=>true, 'feGroup'=>'personal', 'tl_class'=>'w50'),
 			'options_callback' => static function ()
 			{
 				return System::getLanguages();
 			},
-			'sql'                     => "varchar(5) NOT NULL default ''"
+			'sql'                     => "varchar(64) NOT NULL default ''"
 		),
 		'groups' => array
 		(

src/Resources/contao/dca/tl_page.php

@@ -17,6 +17,7 @@
 use Contao\CoreBundle\EventListener\DataContainer\PageUrlListener;
 use Contao\CoreBundle\EventListener\Widget\HttpUrlListener;
 use Contao\CoreBundle\Exception\AccessDeniedException;
+use Contao\CoreBundle\Util\LocaleUtil;
 use Contao\DataContainer;
 use Contao\Idna;
 use Contao\Image;
@@ -257,8 +258,20 @@
 			'exclude'                 => true,
 			'search'                  => true,
 			'inputType'               => 'text',
-			'eval'                    => array('mandatory'=>true, 'rgxp'=>'language', 'maxlength'=>5, 'nospace'=>true, 'doNotCopy'=>true, 'tl_class'=>'w50'),
-			'sql'                     => "varchar(5) NOT NULL default ''"
+			'eval'                    => array('mandatory'=>true, 'maxlength'=>64, 'nospace'=>true, 'decodeEntities'=>true, 'doNotCopy'=>true, 'tl_class'=>'w50'),
+			'sql'                     => "varchar(64) NOT NULL default ''",
+			'save_callback'           => array
+			(
+				static function ($value)
+				{
+					// Make sure there is at least a basic language
+					if (!preg_match('/^[a-z]{2,}/i', $value)) {
+						throw new \RuntimeException($GLOBALS['TL_LANG']['ERR']['language']);
+					}
+
+					return LocaleUtil::canonicalize($value);
+				}
+			)
 		),
 		'robots' => array
 		(

src/Resources/contao/dca/tl_user.php

@@ -13,6 +13,7 @@
 use Contao\BackendUser;
 use Contao\Config;
 use Contao\CoreBundle\Exception\AccessDeniedException;
+use Contao\CoreBundle\Util\LocaleUtil;
 use Contao\DataContainer;
 use Contao\Image;
 use Contao\Input;
@@ -172,16 +173,16 @@
 		),
 		'language' => array
 		(
-			'default'                 => str_replace('-', '_', $GLOBALS['TL_LANGUAGE']),
+			'default'                 => LocaleUtil::formatAsLocale($GLOBALS['TL_LANGUAGE']),
 			'exclude'                 => true,
 			'filter'                  => true,
 			'inputType'               => 'select',
-			'eval'                    => array('rgxp'=>'locale', 'tl_class'=>'w50'),
+			'eval'                    => array('mandatory'=>true, 'tl_class'=>'w50'),
 			'options_callback' => static function ()
 			{
 				return System::getLanguages(true);
 			},
-			'sql'                     => "varchar(5) NOT NULL default ''"
+			'sql'                     => "varchar(64) NOT NULL default ''"
 		),
 		'backendTheme' => array
 		(