[BUGFIX] Ensure routing works for request with chinese letter

Routing for requests with chinese letters failed. As the used symfony components internally uses rawlurldecode() on some places, but this was not done reading from site configuration and reparing route collection, this has been lead to 404 instead of correct page matchings. This patch uses rawurldecode() to prepare route paths correctly for chinese letters and introduce tests to cover these cases. Furthermore it adds some todo's which are out of the scope for this patch. Resolves: #93308 Releases: master, 10.4 Change-Id: I3614cff954d63cd7a960b5d1d5209040019fc083 Reviewed-on: https://review.typo3.org/c/Packages/TYPO3.CMS/+/71513 Tested-by: Jochen <rothjochen@gmail.com> Tested-by: core-ci <typo3@b13.com> Tested-by: Benni Mack <benni@typo3.org> Reviewed-by: Jochen <rothjochen@gmail.com> Reviewed-by: Benni Mack <benni@typo3.org>
TYPO3 · Oct 13, 2021 · 96be2e1 · 96be2e1
1 parent b158399
commit 96be2e1
Show file tree

Hide file tree

Showing 7 changed files with 297 additions and 2 deletions.
diff --git a/typo3/sysext/core/Classes/Routing/SiteMatcher.php b/typo3/sysext/core/Classes/Routing/SiteMatcher.php
@@ -182,12 +182,15 @@ protected function getRouteCollectionForAllSites(): RouteCollection
         $groupedRoutes = [];
         foreach ($this->finder->getAllSites() as $site) {
             // Add the site as entrypoint
+            // @todo Find a way to test only this basic route against chinese characters, as site languages kicking
+            //       always in. Do the rawurldecode() here to to be consistent with language preparations.
             $uri = $site->getBase();
             $route = new Route(
-                ($uri->getPath() ?: '/') . '{tail}',
+                (rawurldecode($uri->getPath()) ?: '/') . '{tail}',
                 ['site' => $site, 'language' => null, 'tail' => ''],
                 array_filter(['tail' => '.*', 'port' => (string)$uri->getPort()]),
                 ['utf8' => true],
+                // @todo Verify if host should here covered with idn_to_ascii() to be consistent with preparation for languages.
                 $uri->getHost() ?: '',
                 $uri->getScheme() === '' ? [] : [$uri->getScheme()]
             );
@@ -197,7 +200,7 @@ protected function getRouteCollectionForAllSites(): RouteCollection
             foreach ($site->getAllLanguages() as $siteLanguage) {
                 $uri = $siteLanguage->getBase();
                 $route = new Route(
-                    ($uri->getPath() ?: '/') . '{tail}',
+                    (rawurldecode($uri->getPath()) ?: '/') . '{tail}',
                     ['site' => $site, 'language' => $siteLanguage, 'tail' => ''],
                     array_filter(['tail' => '.*', 'port' => (string)$uri->getPort()]),
                     ['utf8' => true],

diff --git a/typo3/sysext/frontend/Tests/Functional/SiteHandling/AbstractTestCase.php b/typo3/sysext/frontend/Tests/Functional/SiteHandling/AbstractTestCase.php
@@ -48,6 +48,8 @@ abstract class AbstractTestCase extends FunctionalTestCase
         'FR' => ['id' => 1, 'title' => 'French', 'locale' => 'fr_FR.UTF8', 'iso' => 'fr', 'hrefLang' => 'fr-FR', 'direction' => ''],
         'FR-CA' => ['id' => 2, 'title' => 'Franco-Canadian', 'locale' => 'fr_CA.UTF8', 'iso' => 'fr', 'hrefLang' => 'fr-CA', 'direction' => ''],
         'ES' => ['id' => 3, 'title' => 'Spanish', 'locale' => 'es_ES.UTF8', 'iso' => 'es', 'hrefLang' => 'es-ES', 'direction' => ''],
+        'ZH-CN' => ['id' => 0, 'title' => 'Simplified Chinese', 'locale' => 'zh_CN.UTF-8', 'iso' => 'zh', 'hrefLang' => 'zh-Hans', 'direction' => ''],
+        'ZH' => ['id' => 4, 'title' => 'Simplified Chinese', 'locale' => 'zh_CN.UTF-8', 'iso' => 'zh', 'hrefLang' => 'zh-Hans', 'direction' => ''],
     ];
 
     /**

diff --git a/typo3/sysext/frontend/Tests/Functional/SiteHandling/Fixtures/PlainScenario.yaml b/typo3/sysext/frontend/Tests/Functional/SiteHandling/Fixtures/PlainScenario.yaml
@@ -55,6 +55,7 @@ entities:
           languageVariants:
             - self: {id: 1101, title: 'FR: Welcome', language: 1}
             - self: {id: 1102, title: 'FR-CA: Welcome', language: 2}
+            - self: {id: 1103, title: 'ZH-CN: Welcome', language: 4}
           versionVariants:
             - version: {title: 'EN: Welcome to ACME Inc', workspace: 1}
           entities:
@@ -66,6 +67,13 @@ entities:
                     languageVariants:
                       - self: {title: 'FR-CA: Content Element #1', type: *contentText, language: 2}
               - self: {title: 'EN: Content Element #2', type: *contentText}
+        - self: {id: 1110, title: 'ZH-CN: Welcome Default'}
+          languageVariants:
+            - self: {id: 1111, title: 'FR: Welcome ZH Default', language: 1}
+            - self: {id: 1112, title: 'FR-CA: Welcome ZH Default', language: 2}
+          entities:
+            content:
+              - self: {title: 'EN: Content Element #1', type: *contentText}
         - self: {id: 1200, title: 'EN: Features'}
           children:
             - self: {id: 1210, title: 'EN: Frontend Editing'}

diff --git a/typo3/sysext/frontend/Tests/Functional/SiteHandling/Fixtures/SlugScenario.yaml b/typo3/sysext/frontend/Tests/Functional/SiteHandling/Fixtures/SlugScenario.yaml
@@ -55,6 +55,7 @@ entities:
           languageVariants:
             - self: {id: 1101, title: 'FR: Welcome', language: 1, slug: '/bienvenue', subtitle: 'salut-et-bienvenue'}
             - self: {id: 1102, title: 'FR-CA: Welcome', language: 2, slug: '/bienvenue', subtitle: 'salut-et-bienvenue'}
+            - self: {id: 1103, title: 'ZH-CN: Welcome', language: 4, slug: '/简-bienvenue', subtitle: 'salut-et-bienvenue'}
           versionVariants:
             - version: {title: 'EN: Welcome to ACME Inc', workspace: 1, slug: '/welcome-modified'}
           entities:
@@ -66,6 +67,13 @@ entities:
                     languageVariants:
                       - self: {title: 'FR-CA: Content Element #1', type: *contentText, language: 2}
               - self: {title: 'EN: Content Element #2', type: *contentText}
+        - self: {id: 1110, title: 'ZH-CN: Welcome Default', slug: '/简-bienvenue', subtitle: 'salut-et-bienvenue'}
+          languageVariants:
+            - self: {id: 1111, title: 'FR: Welcome ZH Default', language: 1, slug: '/zh-bienvenue', subtitle: 'salut-et-bienvenue'}
+            - self: {id: 1112, title: 'FR-CA: Welcome ZH Default', language: 2, slug: '/zh-bienvenue', subtitle: 'salut-et-bienvenue'}
+          entities:
+            content:
+              - self: {title: 'EN: Content Element #1', type: *contentText}
         - self: {id: 1200, title: 'EN: Features', slug: '/features'}
           children:
             # This page has a trailing slash as "slug" which is explicitly covered in tests as well

diff --git a/typo3/sysext/frontend/Tests/Functional/SiteHandling/SiteRequestTest.php b/typo3/sysext/frontend/Tests/Functional/SiteHandling/SiteRequestTest.php
@@ -191,6 +191,7 @@ public function pageIsRenderedWithPathsDataProvider(): array
             'en-en/',
             'fr-fr/',
             'fr-ca/',
+            '简/',
         ];
 
         $queries = [
@@ -203,6 +204,8 @@ static function (string $uri) {
                     $expectedPageTitle = 'FR: Welcome';
                 } elseif (str_contains($uri, '/fr-ca/')) {
                     $expectedPageTitle = 'FR-CA: Welcome';
+                } elseif (strpos($uri, '/简/') !== false) {
+                    $expectedPageTitle = 'ZH-CN: Welcome';
                 } else {
                     $expectedPageTitle = 'EN: Welcome';
                 }
@@ -230,6 +233,115 @@ public function pageIsRenderedWithPaths(string $uri, string $expectedPageTitle):
                 $this->buildDefaultLanguageConfiguration('EN', '/en-en/'),
                 $this->buildLanguageConfiguration('FR', '/fr-fr/', ['EN']),
                 $this->buildLanguageConfiguration('FR-CA', '/fr-ca/', ['FR', 'EN']),
+                $this->buildLanguageConfiguration('ZH', '/简/', ['EN']),
+            ]
+        );
+
+        $response = $this->executeFrontendSubRequest(
+            new InternalRequest($uri),
+            $this->internalRequestContext
+        );
+        $responseStructure = ResponseContent::fromString(
+            (string)$response->getBody()
+        );
+
+        self::assertSame(
+            200,
+            $response->getStatusCode()
+        );
+        self::assertSame(
+            $expectedPageTitle,
+            $responseStructure->getScopePath('page/title')
+        );
+    }
+
+    public function pageIsRenderedWithPathsAndChineseDefaultLanguageDataProvider(): array
+    {
+        $domainPaths = [
+            // @todo currently base needs to be defined with domain
+            // '/',
+            'https://website.local/',
+        ];
+
+        $languagePaths = [
+            '简/',
+            'fr-fr/',
+            'fr-ca/',
+        ];
+
+        $queries = [
+            '?id=1110',
+        ];
+
+        return array_map(
+            static function (string $uri) {
+                if (strpos($uri, '/fr-fr/') !== false) {
+                    $expectedPageTitle = 'FR: Welcome ZH Default';
+                } elseif (strpos($uri, '/fr-ca/') !== false) {
+                    $expectedPageTitle = 'FR-CA: Welcome ZH Default';
+                } else {
+                    $expectedPageTitle = 'ZH-CN: Welcome Default';
+                }
+                return [$uri, $expectedPageTitle];
+            },
+            $this->keysFromValues(
+                PermutationUtility::meltStringItems([$domainPaths, $languagePaths, $queries])
+            )
+        );
+    }
+
+    /**
+     * @test
+     * @dataProvider pageIsRenderedWithPathsAndChineseDefaultLanguageDataProvider
+     */
+    public function pageIsRenderedWithPathsAndChineseDefaultLanguage(string $uri, string $expectedPageTitle): void
+    {
+        $this->writeSiteConfiguration(
+            'website-local',
+            $this->buildSiteConfiguration(1000, 'https://website.local/'),
+            [
+                $this->buildDefaultLanguageConfiguration('ZH-CN', '/简/'),
+                $this->buildLanguageConfiguration('FR', '/fr-fr/', ['EN']),
+                $this->buildLanguageConfiguration('FR-CA', '/fr-ca/', ['FR', 'EN']),
+            ]
+        );
+
+        $response = $this->executeFrontendSubRequest(
+            new InternalRequest($uri),
+            $this->internalRequestContext
+        );
+        $responseStructure = ResponseContent::fromString(
+            (string)$response->getBody()
+        );
+
+        self::assertSame(
+            200,
+            $response->getStatusCode()
+        );
+        self::assertSame(
+            $expectedPageTitle,
+            $responseStructure->getScopePath('page/title')
+        );
+    }
+
+    public function pageIsRenderedWithPathsAndChineseBaseDataProvider(): array
+    {
+        return [
+            ['https://website.local/简/简/?id=1110', 'ZH-CN: Welcome Default'],
+        ];
+    }
+
+    /**
+     * @test
+     * @dataProvider pageIsRenderedWithPathsAndChineseBaseDataProvider
+     */
+    public function pageIsRenderedWithPathsAndChineseBase(string $uri, string $expectedPageTitle): void
+    {
+        $this->writeSiteConfiguration(
+            'website-local',
+            $this->buildSiteConfiguration(1000, 'https://website.local/简/'),
+            [
+                $this->buildDefaultLanguageConfiguration('ZH-CN', '/简/'),
             ]
         );
 
@@ -263,6 +375,8 @@ public function pageIsRenderedWithDomainsDataProvider(): array
             'https://website.fr/',
             // Explicitly testing umlaut domains
             'https://wäbsite.ca/',
+            // Explicitly testing chinese character domains
+            'https://website.简/',
             // @todo Implicit strict mode handling when calling non-existent site
             // 'https://website.other/',
         ];
@@ -277,6 +391,8 @@ static function (string $uri) {
                     $expectedPageTitle = 'FR: Welcome';
                 } elseif (str_contains($uri, '.ca/')) {
                     $expectedPageTitle = 'FR-CA: Welcome';
+                } elseif (strpos($uri, '.简/') !== false) {
+                    $expectedPageTitle = 'ZH-CN: Welcome';
                 } else {
                     $expectedPageTitle = 'EN: Welcome';
                 }
@@ -304,6 +420,7 @@ public function pageIsRenderedWithDomains(string $uri, string $expectedPageTitle
                 $this->buildDefaultLanguageConfiguration('EN', 'https://website.us/'),
                 $this->buildLanguageConfiguration('FR', 'https://website.fr/', ['EN']),
                 $this->buildLanguageConfiguration('FR-CA', 'https://wäbsite.ca/', ['FR', 'EN']),
+                $this->buildLanguageConfiguration('ZH', 'https://website.简/', ['EN']),
             ]
         );
 

diff --git a/typo3/sysext/frontend/Tests/Functional/SiteHandling/SlugLinkGeneratorTest.php b/typo3/sysext/frontend/Tests/Functional/SiteHandling/SlugLinkGeneratorTest.php
@@ -707,6 +707,11 @@ public function hierarchicalMenuIsGeneratedDataProvider(): array
                 1100,
                 [
                     ['title' => 'EN: Welcome', 'link' => '/welcome'],
+                    [
+                        'title' => 'ZH-CN: Welcome Default',
+                        // Symfony UrlGenerator, which is used for uri generation, rawurlencodes the url internally.
+                        'link' => '/%E7%AE%80-bienvenue',
+                    ],
                     [
                         'title' => 'EN: Features',
                         'link' => '/features',