Fix infinite loop at illegal sequence [Bug #17729]

As mblen returns -1 on failure, skip the first byte and try the
succeeding bytes in that case.

Close #4281

eval_intern.h

@@ -302,7 +302,16 @@ VALUE rb_ec_backtrace_location_ary(const rb_execution_context_t *ec, long lev, l
 
 #ifndef CharNext		/* defined as CharNext[AW] on Windows. */
 # ifdef HAVE_MBLEN
-#  define CharNext(p) ((p) + mblen((p), RUBY_MBCHAR_MAXSIZE))
+#  define CharNext(p) rb_char_next(p)
+static inline const char *
+rb_char_next(const char *p)
+{
+    if (p) {
+        int len = mblen(p, RUBY_MBCHAR_MAXSIZE);
+        p += len > 0 ? len : 1;
+    }
+    return p;
+}
 # else
 #  define CharNext(p) ((p) + 1)
 # endif

test/ruby/test_rubyoptions.rb

@@ -1089,6 +1089,11 @@ def test_cwd_encoding
     end
   end
 
+  def test_rubylib_invalid_encoding
+    env = {"RUBYLIB"=>"\xFF", "LOCALE"=>"en_US.UTF-8", "LC_ALL"=>"en_US.UTF-8"}
+    assert_ruby_status([env, "-e;"])
+  end
+
   def test_null_script
     skip "#{IO::NULL} is not a character device" unless File.chardev?(IO::NULL)
     assert_in_out_err([IO::NULL], success: true)