Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
branch: master
Fetching contributors…

Octocat-spinner-32-eaf2f5

Cannot retrieve contributors at this time

file 318 lines (283 sloc) 8.209 kb
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317
<?php

namespace utf8;


/**
* Tools for locating/replacing bad bytes in UTF-8 strings.
* The Original Code is Mozilla Communicator client code.
* The Initial Developer of the Original Code is Netscape Communications Corporation.
* Portions created by the Initial Developer are Copyright (C) 1998 the Initial Developer.
* All Rights Reserved.
*
* Ported to PHP by Henri Sivonen (http://hsivonen.iki.fi)
* Slight modifications to fit with phputf8 library by Harry Fuecks (hfuecks gmail com)
*
* @see http://lxr.mozilla.org/seamonkey/source/intl/uconv/src/nsUTF8ToUnicode.cpp
* @see http://lxr.mozilla.org/seamonkey/source/intl/uconv/src/nsUnicodeToUTF8.cpp
* @see http://hsivonen.iki.fi/php-utf8/
* @see utf8_is_valid
* @package php-utf8
* @subpackage utils
*/

/**
* Locates the first bad byte in a UTF-8 string returning it's byte index in the string.
* Optionally it can locate all bad bytes in a UTF-8 string and returns a list of their
* byte index in the string.
*
* PCRE Pattern to locate bad bytes in a UTF-8 string comes from W3 FAQ: Multilingual Forms
* Note: modified to include full ASCII range including control chars
*
* @see http://www.w3.org/International/questions/qa-forms-utf-8
* @param string $str
* @param boolean $first_only
* @return mixed integer byte index or FALSE if no bad found
*/
function badFind($str, $first_only = true)
{
$pos = 0;
$bad_list = array( );

while (preg_match('/'.BAD_UTF_PATTERN.'/S', $str, $matches))
{
$bytes = strlen($matches[0]);

if (isset($matches[2]))
{
if ($first_only)
return $pos;

$bad_list[] = $pos;
}

$pos += $bytes;
$str = substr($str, $bytes);
}

if (!empty($bad_list))
{
return $bad_list;
}

return false;
}

/**
* Strips out any bad bytes from a UTF-8 string and returns the rest.
* Can optionally replace bad bytes with an alternative character.
*
* PCRE Pattern to locate bad bytes in a UTF-8 string comes from W3 FAQ: Multilingual Forms.
* Note: modified to include full ASCII range including control chars
*
* @see http://www.w3.org/International/questions/qa-forms-utf-8
* @param string $str
* @return string
*/
function badClean($str, $replace = false)
{
ob_start();

while (preg_match('/'.BAD_UTF_PATTERN.'/S', $str, $matches))
{
if (!isset($matches[2]))
echo $matches[0];
elseif ($replace !== false && is_string($replace))
echo $replace;

$str = substr($str, strlen($matches[0]));
}

return ob_get_clean();
}


/**
* Return code from utf8_bad_identify() when a five octet sequence is detected.
*
* Note: 5 octets sequences are valid UTF-8 but are not supported by Unicode so do not represent a useful character
*
* @see utf8_bad_identify
*/
define('utf8\BAD_5OCTET', 1);

/**
* Return code from utf8_bad_identify() when a six octet sequence is detected.
*
* Note: 6 octets sequences are valid UTF-8 but are not supported by Unicode so do not represent a useful character
*
* @see utf8_bad_identify
*/
define('utf8\BAD_6OCTET', 2);

/**
* Return code from utf8_bad_identify().
*
* Invalid octet for use as start of multi-byte UTF-8 sequence.
*
* @see utf8_bad_identify
*/
define('utf8\BAD_SEQID', 3);

/**
* Return code from utf8_bad_identify().
*
* From Unicode 3.1, non-shortest form is illegal.
*
* @see utf8_bad_identify
*/
define('utf8\BAD_NONSHORT', 4);

/**
* Return code from utf8_bad_identify().
*
* From Unicode 3.2, surrogate characters are illegal.
*
* @see utf8_bad_identify
*/
define('utf8\BAD_SURROGATE', 5);

/**
* Return code from utf8_bad_identify().
* Codepoints outside the Unicode range are illegal.
* @see utf8_bad_identify
*/
define('utf8\BAD_UNIOUTRANGE', 6);

/**
* Return code from utf8_bad_identify().
*
* Incomplete multi-octet sequence
* Note: this is kind of a "catch-all"
*
* @see utf8_bad_identify
*/
define('utf8\BAD_SEQINCOMPLETE', 7);

/**
* Reports on the type of bad byte found in a UTF-8 string.
*
* Returns a status code on the first bad byte found
*
* @author <hsivonen@iki.fi>
* @param string $str UTF-8 encoded string
* @return mixed $i integer constant describing problem or FALSE if valid UTF-8
* @see utf8_bad_explain
* @see http://hsivonen.iki.fi/php-utf8/
*/
function badIdentify($str, &$i)
{
$mState = 0; // Cached expected number of octets after the current octet
// until the beginning of the next UTF8 character sequence
$mUcs4 = 0; // Cached Unicode character
$mBytes = 1; // Cached expected number of octets in the current sequence

$len = strlen($str);

for ($i = 0; $i < $len; $i++)
{
$in = \ord($str{$i});

if ($mState == 0)
{
// When mState is zero we expect either a US-ASCII character or a multi-octet sequence.
if (0 == (0x80 & ($in)))
{
// US-ASCII, pass straight through.
$mBytes = 1;
}
else if (0xC0 == (0xE0 & ($in)))
{
// First octet of 2 octet sequence
$mUcs4 = ($in);
$mUcs4 = ($mUcs4 & 0x1F) << 6;
$mState = 1;
$mBytes = 2;
}
else if (0xE0 == (0xF0 & ($in)))
{
// First octet of 3 octet sequence
$mUcs4 = ($in);
$mUcs4 = ($mUcs4 & 0x0F) << 12;
$mState = 2;
$mBytes = 3;
}
else if (0xF0 == (0xF8 & ($in)))
{
// First octet of 4 octet sequence
$mUcs4 = ($in);
$mUcs4 = ($mUcs4 & 0x07) << 18;
$mState = 3;
$mBytes = 4;
}
else if (0xF8 == (0xFC & ($in)))
{
/* First octet of 5 octet sequence.
*
* This is illegal because the encoded codepoint must be either
* (a) not the shortest form or
* (b) outside the Unicode range of 0-0x10FFFF.
*/
return BAD_5OCTET;
}
else if (0xFC == (0xFE & ($in)))
{
// First octet of 6 octet sequence, see comments for 5 octet sequence.
return BAD_6OCTET;
}
else
{
// Current octet is neither in the US-ASCII range nor a legal first
// octet of a multi-octet sequence.
return BAD_SEQID;
}
}
else
{
// When mState is non-zero, we expect a continuation of the multi-octet sequence
if (0x80 == (0xC0 & ($in)))
{
// Legal continuation.
$shift = ($mState - 1) * 6;
$tmp = $in;
$tmp = ($tmp & 0x0000003F) << $shift;
$mUcs4 |= $tmp;

/**
* End of the multi-octet sequence. mUcs4 now contains the final
* Unicode codepoint to be output
*/
if (0 == --$mState)
{
// From Unicode 3.1, non-shortest form is illegal
if (((2 == $mBytes) && ($mUcs4 < 0x0080)) || ((3 == $mBytes) && ($mUcs4 < 0x0800)) || ((4 == $mBytes) && ($mUcs4 < 0x10000)) )
return BAD_NONSHORT;
elseif (($mUcs4 & 0xFFFFF800) == 0xD800 ) // From Unicode 3.2, surrogate characters are illegal
return BAD_SURROGATE;
elseif ($mUcs4 > 0x10FFFF ) // Codepoints outside the Unicode range are illegal
return BAD_UNIOUTRANGE;

// Initialize UTF8 cache
$mState = 0;
$mUcs4 = 0;
$mBytes = 1;
}
}
else
{
// ((0xC0 & (*in) != 0x80) && (mState != 0))
// Incomplete multi-octet sequence.
$i--;
return BAD_SEQINCOMPLETE;
}
}
}

// Incomplete multi-octet sequence
if ($mState != 0)
{
$i--;
return BAD_SEQINCOMPLETE;
}

// No bad octets found
$i = null;
return false;
}

/**
* Takes a return code from utf8_bad_identify() are returns a message (in English)
* explaining what the problem is.
*
* @param int $code return code from utf8_bad_identify
* @return mixed string message or FALSE if return code unknown
* @see utf8_bad_identify
*/
function badExplain($code)
{
static $errors;

if (!$errors)
{
$errors = array(
BAD_5OCTET => 'Five octet sequences are valid UTF-8 but are not supported by Unicode',
BAD_6OCTET => 'Six octet sequences are valid UTF-8 but are not supported by Unicode',
BAD_SEQID => 'Invalid octet for use as start of multi-byte UTF-8 sequence',
BAD_NONSHORT => 'From Unicode 3.1, non-shortest form is illegal',
BAD_SURROGATE => 'From Unicode 3.2, surrogate characters are illegal',
BAD_UNIOUTRANGE => 'Codepoints outside the Unicode range are illegal',
BAD_SEQINCOMPLETE => 'Incomplete multi-octet sequence'
);
}

if (isset($errors[$code]))
trigger_error('Unknown error code: '.$errors[$code], E_USER_WARNING);

return false;
}
Something went wrong with that request. Please try again.