From 377279df94a30a623bbb1f86a2f0ce66f8ae1622 Mon Sep 17 00:00:00 2001 From: Takumi Takahashi Date: Wed, 12 Aug 2026 10:23:02 +0900 Subject: [PATCH] string.c: reject UTF-8 sequences forbidden by RFC 3629 `mrb_utf8len` checked only the length implied by the lead byte and the continuation bytes, so it accepted three classes of sequences that the RFC 3629 grammar excludes from UTF-8: - overlong encodings: `C0`/`C1` leads, `E0 80-9F`, `F0 80-8F` - UTF-16 surrogates U+D800 to U+DFFF: `ED A0-BF` - code points above U+10FFFF: `F4 90-BF`, `F5` to `F7` leads Add the lead-specific second byte range checks after the existing continuation byte checks. A rejected sequence takes the same path as every other invalid sequence: it counts one byte and moves on. `mrb_utf8len` backs `String#size`, character indexing, and `String#valid_encoding?`, which now agree with CRuby on such input: ```ruby s = "\xED\xA0\x80" # encodes the surrogate U+D800 s.valid_encoding? # CRuby: false, mruby before: true, after: false s.size # CRuby: 3, mruby before: 1, after: 3 ``` mruby-string-ext already enforces these rules in `utf8code()` behind `String#ord` and in `str_scrub_char_len()` behind `String#scrub`; this brings `mrb_utf8len` in line with them. --- mrbgems/mruby-encoding/test/string.rb | 12 ++++++++++++ src/string.c | 20 ++++++++++++++++++++ test/t/string.rb | 14 ++++++++++++++ 3 files changed, 46 insertions(+) diff --git a/mrbgems/mruby-encoding/test/string.rb b/mrbgems/mruby-encoding/test/string.rb index 6214c8a13e..0ed5b4b00f 100644 --- a/mrbgems/mruby-encoding/test/string.rb +++ b/mrbgems/mruby-encoding/test/string.rb @@ -10,6 +10,18 @@ assert_false "\xfe".valid_encoding? assert_false "あ\xfe".valid_encoding? assert_true "あ\xfe".b.valid_encoding? + # RFC 3629 restrictions + assert_false "\xC0\x80".valid_encoding? # overlong NUL + assert_false "\xC1\xBF".valid_encoding? # overlong (< U+0080) + assert_false "\xE0\x9F\xBF".valid_encoding? # overlong (< U+0800) + assert_false "\xED\xA0\x80".valid_encoding? # surrogate U+D800 + assert_false "\xED\xBF\xBF".valid_encoding? # surrogate U+DFFF + assert_false "\xF0\x8F\xBF\xBF".valid_encoding? # overlong (< U+10000) + assert_false "\xF4\x90\x80\x80".valid_encoding? # above U+10FFFF + assert_false "\xF5\x80\x80\x80".valid_encoding? # above U+10FFFF + assert_true "\u{D7FF}".valid_encoding? # last code point before surrogates + assert_true "\u{E000}".valid_encoding? # first code point after surrogates + assert_true "\u{10FFFF}".valid_encoding? # largest valid code point else assert_true "\xfe".valid_encoding? end diff --git a/src/string.c b/src/string.c index 283e32a356..cc70fae8fd 100644 --- a/src/string.c +++ b/src/string.c @@ -485,6 +485,26 @@ mrb_utf8len(const char* p, const char* e) case 2: if (utf8_islead(p[1])) return 1; } + /* Reject overlong sequences, UTF-16 surrogates, and code points above + U+10FFFF (RFC 3629, Unicode D93b). */ + switch ((unsigned char)p[0]) { + case 0xC0: case 0xC1: /* overlong (< U+0080) */ + return 1; + case 0xE0: /* overlong (< U+0800) */ + if ((unsigned char)p[1] < 0xA0) return 1; + break; + case 0xED: /* surrogate (U+D800..U+DFFF) */ + if ((unsigned char)p[1] > 0x9F) return 1; + break; + case 0xF0: /* overlong (< U+10000) */ + if ((unsigned char)p[1] < 0x90) return 1; + break; + case 0xF4: /* above U+10FFFF */ + if ((unsigned char)p[1] > 0x8F) return 1; + break; + case 0xF5: case 0xF6: case 0xF7: /* above U+10FFFF */ + return 1; + } return len; } diff --git a/test/t/string.rb b/test/t/string.rb index 6239245632..7fd2a139ce 100644 --- a/test/t/string.rb +++ b/test/t/string.rb @@ -639,6 +639,20 @@ def [](*args) assert_equal 2, str[1, 2].size end if UTF8STRING +assert('String#size(UTF-8) counts invalid sequences per byte') do + # RFC 3629: overlong forms, UTF-16 surrogates, and code points above + # U+10FFFF are not characters, so each of their bytes counts on its own + assert_equal 2, "\xC0\x80".size # overlong NUL + assert_equal 3, "\xE0\x9F\xBF".size # overlong (< U+0800) + assert_equal 3, "\xED\xA0\x80".size # surrogate U+D800 + assert_equal 4, "\xF0\x8F\xBF\xBF".size # overlong (< U+10000) + assert_equal 4, "\xF4\x90\x80\x80".size # above U+10FFFF + assert_equal 4, "\xF5\x80\x80\x80".size # above U+10FFFF + assert_equal 1, "\u{D7FF}".size # last code point before surrogates + assert_equal 1, "\u{E000}".size # first code point after surrogates + assert_equal 1, "\u{10FFFF}".size # largest valid code point +end if UTF8STRING + assert('String#slice', '15.2.10.5.34') do # length of args is 1 a = 'abc'.slice(0)