- * The input string. - *
- * @param int $flags [optional]- * A bitmask of one or more of the following flags, which specify how to handle quotes and - * which document type to use. The default is ENT_COMPAT | ENT_HTML401. - *
| Constant Name | - *Description | - *|||||||||||
| ENT_COMPAT | - *Will convert double-quotes and leave single-quotes alone. | - *|||||||||||
| ENT_QUOTES | - *Will convert both double and single quotes. | - *|||||||||||
| ENT_NOQUOTES | - *Will leave both double and single quotes unconverted. | - *|||||||||||
| ENT_HTML401 | - *- * Handle code as HTML 4.01. - * | - *|||||||||||
| ENT_XML1 | + * @return array + */ + protected static function get_broken_utf8_array() + { + return array( + 'ü' => 'ü', + 'ä' => 'ä', + 'ö' => 'ö', + 'Ö' => 'Ö', + 'ß' => 'ß', + 'à ' => 'à', + 'á' => 'á', + 'â' => 'â', + 'ã' => 'ã', + 'ù' => 'ù', + 'ú' => 'ú', + 'û' => 'û', + 'Ù' => 'Ù', + 'Ú' => 'Ú', + 'Û' => 'Û', + 'Ãœ' => 'Ü', + 'ò' => 'ò', + 'ó' => 'ó', + 'ô' => 'ô', + 'è' => 'è', + 'é' => 'é', + 'ê' => 'ê', + 'ë' => 'ë', + 'À' => 'À', + 'Ã' => 'Á', + 'Â' => 'Â', + 'Ã' => 'Ã', + 'Ä' => 'Ä', + 'Ã…' => 'Å', + 'Ç' => 'Ç', + 'È' => 'È', + 'É' => 'É', + 'Ê' => 'Ê', + 'Ë' => 'Ë', + 'ÃŒ' => 'Ì', + 'Ã' => 'Í', + 'ÃŽ' => 'Î', + 'Ã' => 'Ï', + 'Ñ' => 'Ñ', + 'Ã’' => 'Ò', + 'Ó' => 'Ó', + 'Ô' => 'Ô', + 'Õ' => 'Õ', + 'Ø' => 'Ø', + 'Ã¥' => 'å', + 'æ' => 'æ', + 'ç' => 'ç', + 'ì' => 'ì', + 'Ã' => 'í', + 'î' => 'î', + 'ï' => 'ï', + 'ð' => 'ð', + 'ñ' => 'ñ', + 'õ' => 'õ', + 'ø' => 'ø', + 'ý' => 'ý', + 'ÿ' => 'ÿ', + '€' => '€', + ); + } + + /** + * + * UTF-8 version of html_entity_decode() + * + * The reason we are not using html_entity_decode() by itself is because + * while it is not technically correct to leave out the semicolon + * at the end of an entity most browsers will still interpret the entity + * correctly. html_entity_decode() does not convert entities without + * semicolons, so we are left with our own little solution here. Bummer. + * + * Convert all HTML entities to their applicable characters + * + * @link http://php.net/manual/en/function.html-entity-decode.php + * + * @param string $string
| Constant Name | + *Description | + *
| ENT_COMPAT | + *Will convert double-quotes and leave single-quotes alone. | + *
| ENT_QUOTES | + *Will convert both double and single quotes. | + *
| ENT_NOQUOTES | + *Will leave both double and single quotes unconverted. | + *
| ENT_HTML401 | + *+ * Handle code as HTML 4.01. + * | + *
| ENT_XML1 | ** Handle code as XML 1. * | @@ -1898,6 +1876,32 @@ public static function to_utf8($text) return $buf; } + /** + * @param $string + * + * @return bool + */ + public static function isJson($string) + { + if (!$string) { + return false; + } + + if (!is_string($string)) { + return false; + } + + if ( + is_object(json_decode($string)) + && + json_last_error() == JSON_ERROR_NONE + ) { + return true; + } + + return false; + } + /** * Returns part of haystack string from the first occurrence of needle to the end of haystack. * @@ -2040,24 +2044,76 @@ public static function file_get_contents($filename, $flags = null, $context = nu } /** - * is_binary_file + * optimized "mb_detect_encoding()"-function -> with UTF-16 and UTF-32 support * - * @param string $file + * @param string $str * - * @return boolean + * @return bool|string false if we can't detect the string-encoding */ - public static function is_binary_file($file) + public static function str_detect_encoding($str) { - try { - $fp = fopen($file, 'r'); - $block = fread($fp, 512); - fclose($fp); + // init + $encoding = ''; + + // UTF-8 + /** @noinspection PhpUsageOfSilenceOperatorInspection */ + if (substr($str, 0, 3) == @pack('CCC', 0xef, 0xbb, 0xbf)) { + return 'UTF-8'; } - catch (\Exception $e) { - $block = ""; + + // UTF-16 (BE) + /** @noinspection PhpUsageOfSilenceOperatorInspection */ + if (substr($str, 0, 2) == @pack('CC', 0xfe, 0xff)) { + return 'UTF-16BE'; } - return self::is_binary($block); + // UTF-16 (LE) + /** @noinspection PhpUsageOfSilenceOperatorInspection */ + if (substr($str, 0, 2) == @pack('CC', 0xff, 0xfe)) { + return 'UTF-16LE'; + } + + // UTF-32 (BE) + /** @noinspection PhpUsageOfSilenceOperatorInspection */ + if (substr($str, 0, 4) == @pack('CC', 0x00, 0x00, 0xfe, 0xff)) { + return 'UTF-32BE'; + } + + // UTF-32 (LE) + /** @noinspection PhpUsageOfSilenceOperatorInspection */ + if (substr($str, 0, 4) == @pack('CC', 0xff, 0xfe, 0x00, 0x00)) { + return 'UTF32LE'; + } + + if (!$encoding) { + self::checkForSupport(); + + // For UTF-16, UTF-32, UCS2 and UCS4, encoding detection will fail always. + $detectOrder = array( + 'UTF-8', + 'windows-1251', + 'ISO-8859-1', + ); + $encoding = mb_detect_encoding($str, $detectOrder, true); + } + + if (self::is_binary($str)) { + if (self::is_utf16($str) == 1) { + return 'UTF-16LE'; + } else if (self::is_utf16($str) == 2) { + return 'UTF-16BE'; + } else if (self::is_utf32($str) == 1) { + return 'UTF-32LE'; + } else if (self::is_utf32($str) == 2) { + return 'UTF-32BE'; + } + } + + if (!$encoding) { + $encoding = false; + } + + return $encoding; } /** @@ -2086,49 +2142,49 @@ public static function is_binary($input) } /** - * is_utf32 + * is_utf16 * * @param string $string * * @return int|false false if is't not UTF16, 1 for UTF-16LE, 2 for UTF-16BE */ - public static function is_utf32($string) + public static function is_utf16($string) { if (self::is_binary($string)) { self::checkForSupport(); - $maybeUTF32LE = 0; - $test = mb_convert_encoding($string, 'UTF-8', 'UTF-32LE'); + $maybeUTF16LE = 0; + $test = mb_convert_encoding($string, 'UTF-8', 'UTF-16LE'); if ($test !== false && strlen($test) > 1) { - $test2 = mb_convert_encoding($test, 'UTF-32LE', 'UTF-8'); - $test3 = mb_convert_encoding($test2, 'UTF-8', 'UTF-32LE'); + $test2 = mb_convert_encoding($test, 'UTF-16LE', 'UTF-8'); + $test3 = mb_convert_encoding($test2, 'UTF-8', 'UTF-16LE'); if ($test3 == $test) { $stringChars = self::count_chars($string); foreach (self::count_chars($test3) as $test3char => $test3charEmpty) { if (in_array($test3char, $stringChars, true) === true) { - $maybeUTF32LE++; + $maybeUTF16LE++; } } } } - $maybeUTF32BE = 0; - $test = mb_convert_encoding($string, 'UTF-8', 'UTF-32BE'); + $maybeUTF16BE = 0; + $test = mb_convert_encoding($string, 'UTF-8', 'UTF-16BE'); if ($test !== false && strlen($test) > 1) { - $test2 = mb_convert_encoding($test, 'UTF-32BE', 'UTF-8'); - $test3 = mb_convert_encoding($test2, 'UTF-8', 'UTF-32BE'); + $test2 = mb_convert_encoding($test, 'UTF-16BE', 'UTF-8'); + $test3 = mb_convert_encoding($test2, 'UTF-8', 'UTF-16BE'); if ($test3 == $test) { $stringChars = self::count_chars($string); foreach (self::count_chars($test3) as $test3char => $test3charEmpty) { if (in_array($test3char, $stringChars, true) === true) { - $maybeUTF32BE++; + $maybeUTF16BE++; } } } } - if ($maybeUTF32BE != $maybeUTF32LE) { - if ($maybeUTF32LE > $maybeUTF32BE) { + if ($maybeUTF16BE != $maybeUTF16LE) { + if ($maybeUTF16LE > $maybeUTF16BE) { return 1; } else { return 2; @@ -2141,49 +2197,66 @@ public static function is_utf32($string) } /** - * is_utf16 + * returns count of characters used in a string + * + * @param string $str The input string + * + * @return array An associative array of Character as keys and + * their count as values + */ + public static function count_chars($str) //there is no $mode parameters + { + $array = array_count_values(self::split($str)); + + ksort($array); + + return $array; + } + + /** + * is_utf32 * * @param string $string * * @return int|false false if is't not UTF16, 1 for UTF-16LE, 2 for UTF-16BE */ - public static function is_utf16($string) + public static function is_utf32($string) { if (self::is_binary($string)) { self::checkForSupport(); - $maybeUTF16LE = 0; - $test = mb_convert_encoding($string, 'UTF-8', 'UTF-16LE'); + $maybeUTF32LE = 0; + $test = mb_convert_encoding($string, 'UTF-8', 'UTF-32LE'); if ($test !== false && strlen($test) > 1) { - $test2 = mb_convert_encoding($test, 'UTF-16LE', 'UTF-8'); - $test3 = mb_convert_encoding($test2, 'UTF-8', 'UTF-16LE'); + $test2 = mb_convert_encoding($test, 'UTF-32LE', 'UTF-8'); + $test3 = mb_convert_encoding($test2, 'UTF-8', 'UTF-32LE'); if ($test3 == $test) { $stringChars = self::count_chars($string); foreach (self::count_chars($test3) as $test3char => $test3charEmpty) { if (in_array($test3char, $stringChars, true) === true) { - $maybeUTF16LE++; + $maybeUTF32LE++; } } } } - $maybeUTF16BE = 0; - $test = mb_convert_encoding($string, 'UTF-8', 'UTF-16BE'); + $maybeUTF32BE = 0; + $test = mb_convert_encoding($string, 'UTF-8', 'UTF-32BE'); if ($test !== false && strlen($test) > 1) { - $test2 = mb_convert_encoding($test, 'UTF-16BE', 'UTF-8'); - $test3 = mb_convert_encoding($test2, 'UTF-8', 'UTF-16BE'); + $test2 = mb_convert_encoding($test, 'UTF-32BE', 'UTF-8'); + $test3 = mb_convert_encoding($test2, 'UTF-8', 'UTF-32BE'); if ($test3 == $test) { $stringChars = self::count_chars($string); foreach (self::count_chars($test3) as $test3char => $test3charEmpty) { if (in_array($test3char, $stringChars, true) === true) { - $maybeUTF16BE++; + $maybeUTF32BE++; } } } } - if ($maybeUTF16BE != $maybeUTF16LE) { - if ($maybeUTF16LE > $maybeUTF16BE) { + if ($maybeUTF32BE != $maybeUTF32LE) { + if ($maybeUTF32LE > $maybeUTF32BE) { return 1; } else { return 2; @@ -2195,6 +2268,56 @@ public static function is_utf16($string) return false; } + /** + * clean-up a UTF-8 string and show only printable chars at the end + * + * @param $text + * + * @return string + */ + public static function cleanup($text) + { + if (!isset($text[0])) { + return ''; + } + + // init + self::checkForSupport(); + + // fixed ISO <-> UTF-8 Errors + $text = self::fix_simple_utf8($text); + + // remove all none UTF-8 symbols + // && remove diamond question mark (�) + // && remove remove invisible characters (e.g. "\0") + // && remove BOM + // && normalize whitespace chars + $text = self::clean($text, true, true, false); + + return (string)$text; + } + + /** + * is_binary_file + * + * @param string $file + * + * @return boolean + */ + public static function is_binary_file($file) + { + try { + $fp = fopen($file, 'r'); + $block = fread($fp, 512); + fclose($fp); + } + catch (\Exception $e) { + $block = ""; + } + + return self::is_binary($block); + } + /** * Finds the last occurrence of a character in a string within another * @@ -2725,79 +2848,6 @@ public static function is_bom($utf8_chr) return ($utf8_chr === self::bom()); } - /** - * optimized "mb_detect_encoding()"-function -> with UTF-16 and UTF-32 support - * - * @param string $str - * - * @return bool|string false if we can't detect the string-encoding - */ - public static function str_detect_encoding($str) - { - // init - $encoding = ''; - - // UTF-8 - /** @noinspection PhpUsageOfSilenceOperatorInspection */ - if (substr($str, 0, 3) == @pack('CCC', 0xef, 0xbb, 0xbf)) { - return 'UTF-8'; - } - - // UTF-16 (BE) - /** @noinspection PhpUsageOfSilenceOperatorInspection */ - if (substr($str, 0, 2) == @pack('CC', 0xfe, 0xff)) { - return 'UTF-16BE'; - } - - // UTF-16 (LE) - /** @noinspection PhpUsageOfSilenceOperatorInspection */ - if (substr($str, 0, 2) == @pack('CC', 0xff, 0xfe)) { - return 'UTF-16LE'; - } - - // UTF-32 (BE) - /** @noinspection PhpUsageOfSilenceOperatorInspection */ - if (substr($str, 0, 4) == @pack('CC', 0x00, 0x00, 0xfe, 0xff)) { - return 'UTF-32BE'; - } - - // UTF-32 (LE) - /** @noinspection PhpUsageOfSilenceOperatorInspection */ - if (substr($str, 0, 4) == @pack('CC', 0xff, 0xfe, 0x00, 0x00)) { - return 'UTF32LE'; - } - - if (!$encoding) { - self::checkForSupport(); - - // For UTF-16, UTF-32, UCS2 and UCS4, encoding detection will fail always. - $detectOrder = array( - 'UTF-8', - 'windows-1251', - 'ISO-8859-1', - ); - $encoding = mb_detect_encoding($str, $detectOrder, true); - } - - if (self::is_binary($str)) { - if (self::is_utf16($str) == 1) { - return 'UTF-16LE'; - } else if (self::is_utf16($str) == 2) { - return 'UTF-16BE'; - } else if (self::is_utf32($str) == 1) { - return 'UTF-32LE'; - } else if (self::is_utf32($str) == 2) { - return 'UTF-32BE'; - } - } - - if (!$encoding) { - $encoding = false; - } - - return $encoding; - } - /** * returns the Byte Order Mark Character * @@ -3039,23 +3089,6 @@ public static function hex_to_int($str) return 0; } - /** - * returns count of characters used in a string - * - * @param string $str The input string - * - * @return array An associative array of Character as keys and - * their count as values - */ - public static function count_chars($str) //there is no $mode parameters - { - $array = array_count_values(self::split($str)); - - ksort($array); - - return $array; - } - /** * reverses characters order in the string * diff --git a/tests/UTF8Test.php b/tests/UTF8Test.php index 5d6fbc05..c06e4c3e 100644 --- a/tests/UTF8Test.php +++ b/tests/UTF8Test.php @@ -1379,6 +1379,7 @@ public function testUrldecode() "საქართველო" => "საქართველო", "Björk Guðmundsdóttir" => "Björk Guðmundsdóttir", "宮崎 駿" => "宮崎 駿", + '%7B"recipe_id"%3A-1%2C"recipe_created"%3A""%2C"recipe_title"%3A"FSDFSDF"%2C"recipe_description"%3A""%2C"recipe_yield"%3A0%2C"recipe_prepare_time"%3A"fast"%2C"recipe_image"%3A""%2C"recipe_legal"%3A0%2C"recipe_license"%3A0%2C"recipe_category_id"%3A%5B%5D%2C"recipe_category_name"%3A%5B%5D%2C"recipe_variety_id"%3A%5B%5D%2C"recipe_variety_name"%3A%5B%5D%2C"recipe_tag_id"%3A%5B%5D%2C"recipe_tag_name"%3A%5B%5D%2C"recipe_instruction_id"%3A%5B%5D%2C"recipe_instruction_text"%3A%5B%5D%2C"recipe_ingredient_id"%3A%5B%5D%2C"recipe_ingredient_name"%3A%5B%5D%2C"recipe_ingredient_amount"%3A%5B%5D%2C"recipe_ingredient_unit"%3A%5B%5D%2C"errorArray"%3A%7B"recipe_legal"%3A"error"%2C"recipe_license"%3A"error"%2C"recipe_description"%3A"error"%2C"recipe_yield"%3A"error"%2C"recipe_category_name"%3A"error"%2C"recipe_tag_name"%3A"error"%2C"recipe_instruction_text"%3A"error"%2C"recipe_ingredient_amount"%3A"error"%2C"recipe_ingredient_unit"%3A"error"%7D%2C"errorMessage"%3A"%5B%5BBitte+f%5Cu00fclle+die+rot+markierten+Felder+korrekt+aus.%5D%5D"%2C"db"%3A%7B"query_count"%3A15%7D%7D' => '{"recipe_id":-1,"recipe_created":"","recipe_title":"FSDFSDF","recipe_description":"","recipe_yield":0,"recipe_prepare_time":"fast","recipe_image":"","recipe_legal":0,"recipe_license":0,"recipe_category_id":[],"recipe_category_name":[],"recipe_variety_id":[],"recipe_variety_name":[],"recipe_tag_id":[],"recipe_tag_name":[],"recipe_instruction_id":[],"recipe_instruction_text":[],"recipe_ingredient_id":[],"recipe_ingredient_name":[],"recipe_ingredient_amount":[],"recipe_ingredient_unit":[],"errorArray":{"recipe_legal":"error","recipe_license":"error","recipe_description":"error","recipe_yield":"error","recipe_category_name":"error","recipe_tag_name":"error","recipe_instruction_text":"error","recipe_ingredient_amount":"error","recipe_ingredient_unit":"error"},"errorMessage":"[[Bitte f\u00fclle die rot markierten Felder korrekt aus.]]","db":{"query_count":15}}', ); foreach ($testArray as $before => $after) {