[TASK] Use native idn functionality

It is safe to use idn_to_ascii() these days due to symfony's polyfill functionality. This allows us to remove the dependency "algo26-matthias/idna-convert". Also, all other places now use the native idn_to_ascii() call, which could speed up performance. The wrapper call GeneralUtility::idnaEncode() can then safely be deprecated. used composer command: composer remove algo26-matthias/idna-convert Resolves: #87894 Releases: master Change-Id: I85aa6f39b8ff5ac171cd73218ed1144a56d9f724 Reviewed-on: https://review.typo3.org/c/60234 Reviewed-by: Benjamin Franzke <bfr@qbus.de> Reviewed-by: Stephan Großberndt <stephan.grossberndt@typo3.org> Reviewed-by: Georg Ringer <georg.ringer@gmail.com> Tested-by: TYPO3com <noreply@typo3.com> Tested-by: Benjamin Franzke <bfr@qbus.de> Tested-by: Georg Ringer <georg.ringer@gmail.com>
TYPO3 · Mar 16, 2019 · d5dd942 · d5dd942
1 parent 2cb7009
commit d5dd942
Show file tree

Hide file tree

Showing 13 changed files with 183 additions and 157 deletions.
diff --git a/composer.json b/composer.json
@@ -37,7 +37,6 @@
 		"ext-pcre": "*",
 		"ext-session": "*",
 		"ext-xml": "*",
-		"algo26-matthias/idna-convert": "1.1.0",
 		"cogpowered/finediff": "~0.3.1",
 		"doctrine/annotations": "^1.3",
 		"doctrine/dbal": "~2.8.0",

diff --git a/composer.lock b/composer.lock
diff --git a/typo3/sysext/core/Classes/DataHandling/DataHandler.php b/typo3/sysext/core/Classes/DataHandling/DataHandler.php
@@ -2600,7 +2600,7 @@ public function checkValue_input_Eval($value, $evalArray, $is_in, string $table
                     break;
                 case 'domainname':
                     if (!preg_match('/^[a-z0-9.\\-]*$/i', $value)) {
-                        $value = GeneralUtility::idnaEncode($value);
+                        $value = (string)idn_to_ascii($value, IDNA_DEFAULT, INTL_IDNA_VARIANT_UTS46);
                     }
                     break;
                 case 'email':

diff --git a/typo3/sysext/core/Classes/Mail/MailMessage.php b/typo3/sysext/core/Classes/Mail/MailMessage.php
@@ -257,15 +257,15 @@ protected function idnaEncodeAddress($email)
         if (!is_string($email)) {
             return $email;
         }
-        // Split on the last "@" since adresses like "foo@bar"@example.org are valid
+        // Split on the last "@" since addresses like "foo@bar"@example.org are valid
         $atPosition = strrpos($email, '@');
         if (!$atPosition || $atPosition + 1 === strlen($email)) {
             // Return if no @ found or it is placed at the very beginning or end of the email
             return $email;
         }
         $domain = substr($email, $atPosition + 1);
         $local = substr($email, 0, $atPosition);
-        $domain = \TYPO3\CMS\Core\Utility\GeneralUtility::idnaEncode($domain);
+        $domain = (string)idn_to_ascii($domain, IDNA_DEFAULT, INTL_IDNA_VARIANT_UTS46);
 
         return $local . '@' . $domain;
     }

diff --git a/typo3/sysext/core/Classes/Utility/GeneralUtility.php b/typo3/sysext/core/Classes/Utility/GeneralUtility.php
@@ -80,20 +80,6 @@ class GeneralUtility
      */
     protected static $applicationContext;
 
-    /**
-     * IDNA string cache
-     *
-     * @var array<string>
-     */
-    protected static $idnaStringCache = [];
-
-    /**
-     * IDNA converter
-     *
-     * @var \Mso\IdnaConvert\IdnaConvert
-     */
-    protected static $idnaConverter;
-
     /**
      * A list of supported CGI server APIs
      * NOTICE: This is a duplicate of the SAME array in SystemEnvironmentBuilder
@@ -856,9 +842,7 @@ public static function splitCalc($string, $operators)
      *
      * http://tools.ietf.org/html/rfc3696
      * International characters are allowed in email. So the whole address needs
-     * to be converted to punicode before passing it to filter_var(). We convert
-     * the user- and domain part separately to increase the chance of hitting an
-     * entry in self::$idnaStringCache.
+     * to be converted to punicode before passing it to filter_var().
      *
      * Also the @ sign may appear multiple times in an address. If not used as
      * a boundary marker between the user- and domain part, it must be escaped
@@ -883,9 +867,8 @@ public static function validEmail($email)
         $domain = substr($email, $atPosition + 1);
         $user = substr($email, 0, $atPosition);
         if (!preg_match('/^[a-z0-9.\\-]*$/i', $domain)) {
-            try {
-                $domain = self::idnaEncode($domain);
-            } catch (\InvalidArgumentException $exception) {
+            $domain = idn_to_ascii($domain, IDNA_DEFAULT, INTL_IDNA_VARIANT_UTS46);
+            if ($domain === false) {
                 return false;
             }
         }
@@ -897,17 +880,26 @@ public static function validEmail($email)
      *
      * @param string $value
      * @return string An ASCII encoded (punicode) string
+     * @deprecated since TYPO3 v10.0, will be removed in TYPO3 v11.0, use PHP's native idn_to_ascii($domain, IDNA_DEFAULT, INTL_IDNA_VARIANT_UTS46) function directly.
      */
     public static function idnaEncode($value)
     {
-        if (isset(self::$idnaStringCache[$value])) {
-            return self::$idnaStringCache[$value];
-        }
-        if (!self::$idnaConverter) {
-            self::$idnaConverter = new \Mso\IdnaConvert\IdnaConvert(['idn_version' => 2008]);
+        trigger_error(__CLASS__ . ':' . __METHOD__ . ' will be removed in TYPO3 v11.0. Use PHPs native "idn_to_ascii($domain, IDNA_DEFAULT, INTL_IDNA_VARIANT_UTS46)" function directly instead.', E_USER_DEPRECATED);
+        // Early return in case input is not a string or empty
+        if (!is_string($value) || empty($value)) {
+            return (string)$value;
+        }
+        // Split on the last "@" since addresses like "foo@bar"@example.org are valid where the only focus
+        // is an email address
+        $atPosition = strrpos($value, '@');
+        if ($atPosition !== false) {
+            $domain = substr($value, $atPosition + 1);
+            $local = substr($value, 0, $atPosition);
+            $domain = (string)idn_to_ascii($domain, IDNA_DEFAULT, INTL_IDNA_VARIANT_UTS46);
+            // Return if no @ found or it is placed at the very beginning or end of the email
+            return $local . '@' . $domain;
         }
-        self::$idnaStringCache[$value] = self::$idnaConverter->encode($value);
-        return self::$idnaStringCache[$value];
+        return (string)idn_to_ascii($value, IDNA_DEFAULT, INTL_IDNA_VARIANT_UTS46);
     }
 
     /**
@@ -990,11 +982,11 @@ public static function isValidUrl($url)
             return false;
         }
         if (isset($parsedUrl['host']) && !preg_match('/^[a-z0-9.\\-]*$/i', $parsedUrl['host'])) {
-            try {
-                $parsedUrl['host'] = self::idnaEncode($parsedUrl['host']);
-            } catch (\InvalidArgumentException $exception) {
+            $host = idn_to_ascii($parsedUrl['host'], IDNA_DEFAULT, INTL_IDNA_VARIANT_UTS46);
+            if ($host === false) {
                 return false;
             }
+            $parsedUrl['host'] = $host;
         }
         return filter_var(HttpUtility::buildUrl($parsedUrl), FILTER_VALIDATE_URL) !== false;
     }
@@ -3638,7 +3630,6 @@ public static function purgeInstances()
     public static function flushInternalRuntimeCaches()
     {
         self::$indpEnvCache = [];
-        self::$idnaStringCache = [];
     }
 
     /**

diff --git a/...e/Documentation/Changelog/master/Deprecation-87894-GeneralUtilityidnaEncode.rst b/...e/Documentation/Changelog/master/Deprecation-87894-GeneralUtilityidnaEncode.rst
@@ -0,0 +1,39 @@
+.. include:: ../../Includes.txt
+
+================================================
+Deprecation: #87894 - GeneralUtility::idnaEncode
+================================================
+
+See :issue:`87894`
+
+Description
+===========
+
+PHP has the native function :php:`idn_to_ascii($domain, IDNA_DEFAULT, INTL_IDNA_VARIANT_UTS46)` for converting UTF-8 based domains to ascii-based ("punicode")
+which is available in all supported PHP versions using :php:`"symfony/polyfill-intl-idn"`.
+
+For this reason the method :php:`GeneralUtility::idnaEncode()` has been marked as deprecated.
+
+
+Impact
+======
+
+Calling :php:`GeneralUtility::idnaEncode()` directly will trigger a deprecation message.
+
+
+Affected Installations
+======================
+
+Any TYPO3 installation with third-party extensions calling this method.
+
+
+Migration
+=========
+
+Use :php:`idn_to_ascii($domain, IDNA_DEFAULT, INTL_IDNA_VARIANT_UTS46);` instead.
+
+Please be aware that in contrary to :php:`GeneralUtility::idnaEncode()` the native PHP function only works on domain names, not email addresses or
+similar. In order to encode email addresses split the address at the last :php:`'@'` and use :php:`idn_to_ascii()` on that last part.
+Also, if there is an error in converting a string, a bool :php:`false` is returned.
+
+.. index:: PHP-API, FullyScanned, ext:core
diff --git a/...elog/master/Important-87894-RemovedPHPDependencyAlgo26-matthiasidna-convert.rst b/...elog/master/Important-87894-RemovedPHPDependencyAlgo26-matthiasidna-convert.rst
@@ -0,0 +1,20 @@
+.. include:: ../../Includes.txt
+
+=======================================================================
+Important: #87894 - Removed PHP dependency algo26-matthias/idna-convert
+=======================================================================
+
+See :issue:`87894`
+
+Description
+===========
+
+PHP has native functions for converting UTF-8 based domains to ascii-based ("punicode"), which
+can be used directly when the PHP extension "intl" is installed. For servers with PHP packages which
+do not have the PHP extension "intl" installed, the symfony polyfill package "symfony/polyfill-intl-idn"
+is available, allowing to use native PHP functionality in the TYPO3 code base.
+
+For this reason the PHP dependency "algo26-matthias/idna-convert" is no longer necessary and
+has been removed.
+
+.. index:: PHP-API, ext:core
diff --git a/typo3/sysext/core/Tests/Unit/Utility/GeneralUtilityTest.php b/typo3/sysext/core/Tests/Unit/Utility/GeneralUtilityTest.php
@@ -4687,66 +4687,6 @@ public function xml2ArrayHandlesAttributeTypes(string $input, $expected)
         $this->assertSame($expected, $result['index']['vDEF']);
     }
 
-    /**
-     * @test
-     * @dataProvider idnaEncodeDataProvider
-     * @param $actual
-     * @param $expected
-     */
-    public function idnaEncodeConvertsUnicodeCharsToASCIIString($actual, $expected)
-    {
-        $result = GeneralUtility::idnaEncode($actual);
-        $this->assertSame($expected, $result);
-    }
-
-    /**
-     * Data provider for method idnaEncode in GeneralUtility class.
-     * IDNA converter has to convert special chars (UTF-8) to ASCII compatible chars.
-     *
-     * @returns array
-     */
-    public function idnaEncodeDataProvider()
-    {
-        return [
-            'empty string' => [
-                '',
-                ''
-            ],
-            'null value' => [
-                null,
-                ''
-            ],
-            'string with ascii chars' => [
-                'example',
-                'example'
-            ],
-            'domain (1) with utf8 chars' => [
-                'dömäin.example',
-                'xn--dmin-moa0i.example'
-            ],
-            'domain (2) with utf8 chars' => [
-                'äaaa.example',
-                'xn--aaa-pla.example'
-            ],
-            'domain (3) with utf8 chars' => [
-                'déjà.vu.example',
-                'xn--dj-kia8a.vu.example'
-            ],
-            'domain (4) with utf8 chars' => [
-                'foo.âbcdéf.example',
-                'foo.xn--bcdf-9na9b.example'
-            ],
-            'domain with utf8 char (german umlaut)' => [
-                'exömple.com',
-                'xn--exmple-xxa.com'
-            ],
-            'email with utf8 char (german umlaut)' => [
-                'joe.doe@dömäin.de',
-                'joe.doe@xn--dmin-moa0i.de'
-            ]
-        ];
-    }
-
     public function splitHeaderLinesDataProvider(): array
     {
         return [