Skip to content

Commit

Permalink
Merge pull request #1076 from ma8ma/implement-misc-sanitize_numeric_c…
Browse files Browse the repository at this point in the history 
…harref

Implement MISC::sanitize_numeric_charref()
  • Loading branch information
@ma8ma
ma8ma committed Dec 10, 2022
2 parents 1cc18c0 + 5b7a1b1 commit 31d2e10
Show file tree
Hide file tree
Showing 3 changed files with 169 additions and 46 deletions.
96 changes: 50 additions & 46 deletions src/jdlib/miscutil.cpp
View file
Expand Up @@ -1589,51 +1589,6 @@ static char32_t transform_7f_9f( char32_t raw_point )
}
}

//
// コードポイントが数値文字参照としてはエラーなら規定の値へ変換する
// 例えばサロゲートペアは'REPLACEMENT CHARACTER' (U+FFFD)を返す
//
// 参考文献 : Numeric character reference end state (HTML 5.3)
// https://www.w3.org/TR/html53/syntax.html#numeric-character-reference-end-state
//
static char32_t sanitize_numeric_character_reference( char32_t raw_point )
{
// NOTE: 記号や絵文字を速やかに処理できるよう順番が組まれている

bool parse_error = false;

// 基本多言語面(BMP)をおおまかにチェック
if( 0x009F < raw_point && raw_point < 0xD800 ) return raw_point;
// 特定のbitパターンの非文字と符号空間の範囲をチェック
else if( ( raw_point & 0xFFFE ) == 0xFFFE || raw_point > 0x10FFFF ) parse_error = true;
// bitパターンを除いたらBMPの一部と追加多言語面(SMP)以降をチェック
else if( 0xFDEF < raw_point ) return raw_point;
// サロゲートペアはエラー
else if( 0xD800 <= raw_point && raw_point <= 0xDFFF ) parse_error = true;

// 基本ラテン文字をチェック
else if( 0x001F < raw_point && raw_point < 0x007F ) return raw_point;
// 特定の変換が必要なコードポイントをチェック
else if( 0x007F <= raw_point && raw_point <= 0x009F ) return transform_7f_9f( raw_point );
// 最後に制御文字と非文字をチェック
// サロゲートペアは他の値より入力される可能性が高いので処理を優先している
else if( raw_point <= 0x0008 // Control character
|| raw_point == 0x000B // Control character (Vertical tab)
|| ( 0x000D <= raw_point && raw_point <= 0x001F ) // Control character
|| ( 0xFDD0 <= raw_point // && raw_point <= 0xFDEF の境界は上でチェックしているので不要
) // Noncharacters
) {
parse_error = true;
}

if( parse_error ) {
#ifdef _DEBUG
std::cout << "Parse error for numeric character reference... " << raw_point << std::endl;
#endif
return 0xFFFD; // REPLACEMENT CHARACTER
}
return raw_point;
}

//
// 「&#数字;」形式の数字参照文字列を数字(int)に変換する
Expand All @@ -1659,7 +1614,56 @@ char32_t MISC::decode_spchar_number( const char* in_char, const int offset, cons

const int base{ offset == 2 ? 10 : 16 };
const char32_t uch = static_cast<char32_t>( std::strtoul( str_num, nullptr, base ) );
return sanitize_numeric_character_reference( uch );
return MISC::sanitize_numeric_charref( uch );
}


/** @brief コードポイントが数値文字参照の無効・解析エラーなら規定の値へ変換する
*
* @details WHATWG の仕様と異なり
* C0/C1 controls のうち変換表にリストされていないものは U+FFFD へ変換する
* @param[in] uch 数値文字参照を解析して得た値
* @param[out] high_surrogate `uch` が上位サロゲート(U+D800 - U+DBFF)のときは代入して返す (nullable)
* @return 変換した結果
*
* @remarks 参考文献 : Numeric character reference end state (WHATWG) @n
* https://html.spec.whatwg.org/multipage/parsing.html#numeric-character-reference-end-state
*/
char32_t MISC::sanitize_numeric_charref( const char32_t uch, char32_t* high_surrogate )
{
constexpr char32_t replace = 0xFFFD; // REPLACEMENT CHARACTER
if( uch >= 0xD800 ) {
// 特定のbitパターンの非文字をチェック
if( ( uch & 0xFFFE ) == 0xFFFE ) return replace;
// 上位サロゲート (U+D800 - U+DBFF)
else if( uch < 0xDC00 ) {
if( high_surrogate ) *high_surrogate = uch;
return replace;
}
// 下位サロゲート (U+DC00 - U+DFFF)
else if( uch < 0xE000 ) return replace;
else if( uch < 0xFDD0 ) return uch;
// 非文字 (noncharacters, U+FDD0 - U+FDEF)
else if( uch < 0xFDF0 ) return replace;
else if( uch < 0x110000 ) return uch;
// Unicodeの範囲外
return replace;
}
else if( uch < 0x00A0 ) {
// C1 Controls
if( 0x007F <= uch ) return transform_7f_9f( uch );
// ASCII printable characters
else if( 0x0020 <= uch ) return uch;
else if( 0x000D <= uch ) return replace;
// FORM FEED (FF)
else if( 0x000C == uch ) return uch;
// LINE TABULATION (VT)
else if( 0x000B == uch ) return replace;
// CHARACTER TABULATION (HT) and LINE FEED (LF)
else if( 0x0009 <= uch ) return uch;
return replace;
}
return uch;
}


Expand Down
3 changes: 3 additions & 0 deletions src/jdlib/miscutil.h
View file
Expand Up @@ -216,6 +216,9 @@ namespace MISC
//
char32_t decode_spchar_number( const char* in_char, const int offset, const int lng );

// コードポイントが数値文字参照の無効・解析エラーなら規定の値へ変換する
char32_t sanitize_numeric_charref( const char32_t uch, char32_t* high_surrogate = nullptr );

// str に含まれる「&#数字;」形式の数字参照文字列を全てユニーコード文字に変換する
std::string decode_spchar_number( const std::string& str );

Expand Down
116 changes: 116 additions & 0 deletions test/gtest_jdlib_miscutil.cpp
View file
Expand Up @@ -1821,6 +1821,122 @@ TEST_F(UrlDecodeTest, out_of_range_segments)
}


class MISC_SanitizeNumericCharrefTest : public ::testing::Test {};

TEST_F(MISC_SanitizeNumericCharrefTest, null_character)
{
char32_t out = 0;
EXPECT_EQ( 0xFFFD, MISC::sanitize_numeric_charref( 0, &out ) );
EXPECT_EQ( 0, out );
}

TEST_F(MISC_SanitizeNumericCharrefTest, carriage_return)
{
char32_t out = 0;
EXPECT_EQ( 0xFFFD, MISC::sanitize_numeric_charref( 0x000D, &out ) );
EXPECT_EQ( 0, out );
}

TEST_F(MISC_SanitizeNumericCharrefTest, ascii_whitespace)
{
char32_t out = 0;
EXPECT_EQ( 0x0009, MISC::sanitize_numeric_charref( 0x0009, &out ) );
EXPECT_EQ( 0, out );

out = 0;
EXPECT_EQ( 0x000A, MISC::sanitize_numeric_charref( 0x000A, &out ) );
EXPECT_EQ( 0, out );

out = 0;
EXPECT_EQ( 0x000C, MISC::sanitize_numeric_charref( 0x000C, &out ) );
EXPECT_EQ( 0, out );

out = 0;
EXPECT_EQ( 0x0020, MISC::sanitize_numeric_charref( 0x0020, &out ) );
EXPECT_EQ( 0, out );
}

TEST_F(MISC_SanitizeNumericCharrefTest, out_of_range)
{
char32_t out = 0;
EXPECT_EQ( 0xFFFD, MISC::sanitize_numeric_charref( 0x110000, &out ) );
EXPECT_EQ( 0, out );
EXPECT_EQ( 0xFFFD, MISC::sanitize_numeric_charref( 0x1000000, nullptr ) );
}

TEST_F(MISC_SanitizeNumericCharrefTest, high_surrogate)
{
// 数が多いため網羅していない
char32_t out = 0;
EXPECT_EQ( 0xFFFD, MISC::sanitize_numeric_charref( 0xD800, &out ) );
EXPECT_EQ( 0xD800, out );

out = 0;
EXPECT_EQ( 0xFFFD, MISC::sanitize_numeric_charref( 0xDBFF, &out ) );
EXPECT_EQ( 0xDBFF, out );
}

TEST_F(MISC_SanitizeNumericCharrefTest, low_surrogate)
{
// 数が多いため網羅していない
char32_t out = 0;
EXPECT_EQ( 0xFFFD, MISC::sanitize_numeric_charref( 0xDC00, &out ) );
EXPECT_EQ( 0, out );

out = 0;
EXPECT_EQ( 0xFFFD, MISC::sanitize_numeric_charref( 0xDFFF, &out ) );
EXPECT_EQ( 0, out );
}

TEST_F(MISC_SanitizeNumericCharrefTest, noncharacter)
{
// 数が多いため網羅していない
constexpr char32_t nonchars[] = { 0xFDD0, 0xFDEF, 0xFFFE, 0xFFFF,
0x1FFFE, 0x1FFFF, 0x10FFFE, 0x10FFFF };
for( const char32_t uch : nonchars ) {
char32_t out = 0;
EXPECT_EQ( 0xFFFD, MISC::sanitize_numeric_charref( uch, &out ) );
EXPECT_EQ( 0, out );
}
}

TEST_F(MISC_SanitizeNumericCharrefTest, between_u007F_and_u009F)
{
EXPECT_EQ( 0xFFFD, MISC::sanitize_numeric_charref( 0x007F, nullptr ) );
EXPECT_EQ( 0x20AC, MISC::sanitize_numeric_charref( 0x0080, nullptr ) );
EXPECT_EQ( 0xFFFD, MISC::sanitize_numeric_charref( 0x0081, nullptr ) );
EXPECT_EQ( 0x201A, MISC::sanitize_numeric_charref( 0x0082, nullptr ) );
EXPECT_EQ( 0x0192, MISC::sanitize_numeric_charref( 0x0083, nullptr ) );
EXPECT_EQ( 0x201E, MISC::sanitize_numeric_charref( 0x0084, nullptr ) );
EXPECT_EQ( 0x2026, MISC::sanitize_numeric_charref( 0x0085, nullptr ) );
EXPECT_EQ( 0x2020, MISC::sanitize_numeric_charref( 0x0086, nullptr ) );
EXPECT_EQ( 0x2021, MISC::sanitize_numeric_charref( 0x0087, nullptr ) );
EXPECT_EQ( 0x02C6, MISC::sanitize_numeric_charref( 0x0088, nullptr ) );
EXPECT_EQ( 0x2030, MISC::sanitize_numeric_charref( 0x0089, nullptr ) );
EXPECT_EQ( 0x0160, MISC::sanitize_numeric_charref( 0x008A, nullptr ) );
EXPECT_EQ( 0x2039, MISC::sanitize_numeric_charref( 0x008B, nullptr ) );
EXPECT_EQ( 0x0152, MISC::sanitize_numeric_charref( 0x008C, nullptr ) );
EXPECT_EQ( 0x017D, MISC::sanitize_numeric_charref( 0x008E, nullptr ) );
EXPECT_EQ( 0xFFFD, MISC::sanitize_numeric_charref( 0x008F, nullptr ) );
EXPECT_EQ( 0xFFFD, MISC::sanitize_numeric_charref( 0x0090, nullptr ) );
EXPECT_EQ( 0x2018, MISC::sanitize_numeric_charref( 0x0091, nullptr ) );
EXPECT_EQ( 0x2019, MISC::sanitize_numeric_charref( 0x0092, nullptr ) );
EXPECT_EQ( 0x201C, MISC::sanitize_numeric_charref( 0x0093, nullptr ) );
EXPECT_EQ( 0x201D, MISC::sanitize_numeric_charref( 0x0094, nullptr ) );
EXPECT_EQ( 0x2022, MISC::sanitize_numeric_charref( 0x0095, nullptr ) );
EXPECT_EQ( 0x2013, MISC::sanitize_numeric_charref( 0x0096, nullptr ) );
EXPECT_EQ( 0x2014, MISC::sanitize_numeric_charref( 0x0097, nullptr ) );
EXPECT_EQ( 0x02DC, MISC::sanitize_numeric_charref( 0x0098, nullptr ) );
EXPECT_EQ( 0x2122, MISC::sanitize_numeric_charref( 0x0099, nullptr ) );
EXPECT_EQ( 0x0161, MISC::sanitize_numeric_charref( 0x009A, nullptr ) );
EXPECT_EQ( 0x203A, MISC::sanitize_numeric_charref( 0x009B, nullptr ) );
EXPECT_EQ( 0x0153, MISC::sanitize_numeric_charref( 0x009C, nullptr ) );
EXPECT_EQ( 0xFFFD, MISC::sanitize_numeric_charref( 0x009D, nullptr ) );
EXPECT_EQ( 0x017E, MISC::sanitize_numeric_charref( 0x009E, nullptr ) );
EXPECT_EQ( 0x0178, MISC::sanitize_numeric_charref( 0x009F, nullptr ) );
}


class AsciiIgnoreCaseFindTest : public ::testing::Test {};

TEST_F(AsciiIgnoreCaseFindTest, empty)
Expand Down

0 comments on commit 31d2e10

Please sign in to comment.