Skip to content

Commit

Permalink
[TASK] Remove dependency to algo26-matthias/idna-convert
Browse files Browse the repository at this point in the history
The third-party library algo26-matthias/idna-convert is not compatible
with PHP 7.4 (used for providing IDNA conversions for umlaut domains).

The classes from v1 are copied in order to have them still available, but we
maintain the code ourselves and reduce the usage to a minimum.

  composer remove "algo26-matthias/idna-convert:"

See https://idnaconv.net/docs.html for detailed upgrade
changes.

Resolves: #90911
Releases: 9.5
Change-Id: I7ac14348677b8b60f92cb8bfa26e26d16a01abee
Reviewed-on: https://review.typo3.org/c/Packages/TYPO3.CMS/+/64011
Tested-by: Manuel Selbach <manuel_selbach@yahoo.de>
Tested-by: Frank Nägler <frank.naegler@typo3.org>
Reviewed-by: Manuel Selbach <manuel_selbach@yahoo.de>
Reviewed-by: Frank Nägler <frank.naegler@typo3.org>
  • Loading branch information
bmack committed Apr 15, 2020
1 parent ccc7eae commit 31c8142
Show file tree
Hide file tree
Showing 18 changed files with 4,074 additions and 65 deletions.
3 changes: 3 additions & 0 deletions Build/Scripts/duplicateExceptionCodeCheck.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@ ignoreFiles+="sysext/extbase/Classes/Core/Bootstrap.php"
ignoreFiles+="sysext/form/Classes/Mvc/Property/Exception/TypeConverterException.php"
ignoreFiles+="sysext/core/Classes/Database/Driver/PDOStatement.php"
ignoreFiles+="sysext/core/Classes/Database/Driver/PDOConnection.php"
ignoreFiles+="sysext/core/Resources/PHP/idna-convert/IdnaConvert.php"
ignoreFiles+="sysext/core/Resources/PHP/idna-convert/UnicodeTranscoder.php"
ignoreFiles+="sysext/core/Resources/PHP/idna-convert/Punycode.php"

foundNewFile=0
oldFilename=""
Expand Down
1 change: 0 additions & 1 deletion composer.json
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@
"ext-pcre": "*",
"ext-session": "*",
"ext-xml": "*",
"algo26-matthias/idna-convert": "^1.1.0",
"cogpowered/finediff": "~0.3.1",
"doctrine/annotations": "^1.7",
"doctrine/dbal": "^2.10",
Expand Down
84 changes: 34 additions & 50 deletions composer.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

27 changes: 17 additions & 10 deletions typo3/sysext/core/Classes/Utility/GeneralUtility.php
Original file line number Diff line number Diff line change
Expand Up @@ -98,13 +98,6 @@ class GeneralUtility
*/
protected static $idnaStringCache = [];

/**
* IDNA converter
*
* @var \Mso\IdnaConvert\IdnaConvert
*/
protected static $idnaConverter;

/**
* A list of supported CGI server APIs
* NOTICE: This is a duplicate of the SAME array in SystemEnvironmentBuilder
Expand Down Expand Up @@ -948,10 +941,24 @@ public static function idnaEncode($value)
if (isset(self::$idnaStringCache[$value])) {
return self::$idnaStringCache[$value];
}
if (!self::$idnaConverter) {
self::$idnaConverter = new \Mso\IdnaConvert\IdnaConvert(['idn_version' => 2008]);
// Early return in case input is not a string or empty
if (!is_string($value) || empty($value)) {
return (string)$value;
}
self::$idnaStringCache[$value] = self::$idnaConverter->encode($value);

// Split on the last "@" since addresses like "foo@bar"@example.org are valid where the only focus
// is an email address
$atPosition = strrpos($value, '@');
if ($atPosition !== false) {
$domain = substr($value, $atPosition + 1);
$local = substr($value, 0, $atPosition);
$domain = (string)HttpUtility::idn_to_ascii($domain);
// Return if no @ found or it is placed at the very beginning or end of the email
self::$idnaStringCache[$value] = $local . '@' . $domain;
return self::$idnaStringCache[$value];
}

self::$idnaStringCache[$value] = (string)HttpUtility::idn_to_ascii($value);
return self::$idnaStringCache[$value];
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
.. include:: ../../Includes.txt

================================================================
Important: #90911 - Package algo26-matthias/idna-convert removed
================================================================

See :issue:`90911`

Description
===========

The TYPO3 core dependency / composer library `algo26-matthias/idna-convert` does not support PHP 7.4
in its currently used version. It has been removed from the composer dependencies and the current used
code is placed into `typo3/sysext/core/Resources/PHP/idna-converter` to support directly usage of
that package.

This makes it possible to use TYPO3 v9 with umlaut domain validation (e.g. also when using EXT:form
with sending an email to someone with a umlaut domain as recipient) in conjunction with TYPO3 v9 and
PHP 7.4.

If the PHP code of the package is used directly by third-party extensions, this will have no further
side effects, as the TYPO3 core still provides the source code, but be aware will not work with
PHP 7.4.

If you like to use the package in a newer version, follow the docs https://idnaconv.net/docs.html
and add the custom PHP code yourself.

If you are using the TYPO3 API `GeneralUtility::idnaEncode()` everything works as before, but now
also with PHP 7.4 support.

.. index:: PHP-API, ext:core
172 changes: 172 additions & 0 deletions typo3/sysext/core/Resources/PHP/idna-convert/EncodingHelper.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
<?php
/**
* Encoding Helper - convert various encodings to / from UTF-8
* @author Matthias Sommerfeld, <mso@phlylabs.de>
* @copyright 2003-2016 phlyLabs Berlin, http://phlylabs.de
* @version 1.0.0 2016-01-08
*/
namespace Mso\IdnaConvert;

class EncodingHelper
{
/**
* Convert a string from any of various encodings to UTF-8
*
* @param string $string String to encode
*[@param string $encoding Encoding; Default: ISO-8859-1]
*[@param bool $safe_mode Safe Mode: if set to TRUE, the original string is retunred on errors]
* @return string|false The encoded string or false on failure
* @since 0.0.1
*/
public static function toUtf8($string = '', $encoding = 'iso-8859-1', $safe_mode = false)
{
$safe = ($safe_mode) ? $string : false;
if (strtoupper($encoding) == 'UTF-8' || strtoupper($encoding) == 'UTF8') {
return $string;
}
if (strtoupper($encoding) == 'ISO-8859-1') {
return \utf8_encode($string);
}
if (strtoupper($encoding) == 'WINDOWS-1252') {
return \utf8_encode(self::map_w1252_iso8859_1($string));
}

if (strtoupper($encoding) == 'UNICODE-1-1-UTF-7') {
$encoding = 'utf-7';
}
if (function_exists('mb_convert_encoding')) {
$conv = @mb_convert_encoding($string, 'UTF-8', strtoupper($encoding));
if ($conv) {
return $conv;
}
}
if (function_exists('iconv')) {
$conv = @iconv(strtoupper($encoding), 'UTF-8', $string);
if ($conv) {
return $conv;
}
}
if (function_exists('libiconv')) {
$conv = @libiconv(strtoupper($encoding), 'UTF-8', $string);
if ($conv) {
return $conv;
}
}

return $safe;
}

/**
* Convert a string from UTF-8 to any of various encodings
*
* @param string $string String to decode
*[@param string $encoding Encoding; Default: ISO-8859-1]
*[@param bool $safe_mode Safe Mode: if set to TRUE, the original string is retunred on errors]
* @return string|false The decoded string or false on failure
* @since 0.0.1
*/
public static function fromUtf8($string = '', $encoding = 'iso-8859-1', $safe_mode = false)
{
$safe = ($safe_mode) ? $string : false;
if (!$encoding) {
$encoding = 'ISO-8859-1';
}
if (strtoupper($encoding) == 'UTF-8' || strtoupper($encoding) == 'UTF8') {
return $string;
}
if (strtoupper($encoding) == 'ISO-8859-1') {
return utf8_decode($string);
}
if (strtoupper($encoding) == 'WINDOWS-1252') {
return self::map_iso8859_1_w1252(utf8_decode($string));
}

if (strtoupper($encoding) == 'UNICODE-1-1-UTF-7') {
$encoding = 'utf-7';
}
if (function_exists('mb_convert_encoding')) {
$conv = @mb_convert_encoding($string, strtoupper($encoding), 'UTF-8');
if ($conv) {
return $conv;
}
}
if (function_exists('iconv')) {
$conv = @iconv('UTF-8', strtoupper($encoding), $string);
if ($conv) {
return $conv;
}
}
if (function_exists('libiconv')) {
$conv = @libiconv('UTF-8', strtoupper($encoding), $string);
if ($conv) {
return $conv;
}
}

return $safe;
}

/**
* Special treatment for our guys in Redmond
* Windows-1252 is basically ISO-8859-1 -- with some exceptions, which get accounted for here
*
* @param string $string Your input in Win1252
* @return string The resulting ISO-8859-1 string
* @since 0.0.1
*/
protected static function map_w1252_iso8859_1($string = '')
{
if ($string == '') {
return '';
}
$return = '';

for ($i = 0; $i < strlen($string); ++$i) {
$c = ord($string{$i});
switch ($c) {
case 129: $return .= chr(252); break;
case 132: $return .= chr(228); break;
case 142: $return .= chr(196); break;
case 148: $return .= chr(246); break;
case 153: $return .= chr(214); break;
case 154: $return .= chr(220); break;
case 225: $return .= chr(223); break;
default: $return .= chr($c);
}
}

return $return;
}

/**
* Special treatment for our guys in Redmond
* Windows-1252 is basically ISO-8859-1 -- with some exceptions, which get accounted for here
*
* @param string $string Your input in ISO-8859-1
* @return string The resulting Win1252 string
* @since 0.0.1
*/
protected static function map_iso8859_1_w1252($string = '')
{
if ($string == '') {
return '';
}

$return = '';
for ($i = 0; $i < strlen($string); ++$i) {
$c = ord($string{$i});
switch ($c) {
case 196: $return .= chr(142); break;
case 214: $return .= chr(153); break;
case 220: $return .= chr(154); break;
case 223: $return .= chr(225); break;
case 228: $return .= chr(132); break;
case 246: $return .= chr(148); break;
case 252: $return .= chr(129); break;
default: $return .= chr($c);
}
}

return $return;
}
}
Loading

0 comments on commit 31c8142

Please sign in to comment.