Check integer overflow in long range

https://hackerone.com/reports/1328463

Author: nobu

ext/cgi/escape/escape.c

@@ -32,12 +32,21 @@ preserve_original_state(VALUE orig, VALUE dest)
     rb_enc_associate(dest, rb_enc_get(orig));
 }
 
+static inline long
+escaped_length(VALUE str)
+{
+    const long len = RSTRING_LEN(str);
+    if (len >= LONG_MAX / HTML_ESCAPE_MAX_LEN) {
+	ruby_malloc_size_overflow(len, HTML_ESCAPE_MAX_LEN);
+    }
+    return len * HTML_ESCAPE_MAX_LEN;
+}
+
 static VALUE
 optimized_escape_html(VALUE str)
 {
     VALUE vbuf;
-    typedef char escape_buf[HTML_ESCAPE_MAX_LEN];
-    char *buf = *ALLOCV_N(escape_buf, vbuf, RSTRING_LEN(str));
+    char *buf = ALLOCV_N(char, vbuf, escaped_length(str));
     const char *cstr = RSTRING_PTR(str);
     const char *end = cstr + RSTRING_LEN(str);
 

test/cgi/test_cgi_util.rb

@@ -104,6 +104,23 @@ def test_cgi_escape_html_dont_freeze
     assert_not_predicate CGI.escapeHTML("Ruby".freeze),   :frozen?
   end
 
+  def test_cgi_escape_html_large
+    ulong_max, size_max = RbConfig::LIMITS.values_at("ULONG_MAX", "SIZE_MAX")
+    return unless ulong_max < size_max # Platforms not concerned
+
+    size = (ulong_max / 6 + 1)
+    begin
+      str = '"' * size
+      escaped = CGI.escapeHTML(str)
+    rescue NoMemoryError
+      omit "Not enough memory"
+    rescue => e
+    end
+    assert_raise_with_message(ArgumentError, /overflow/, ->{"length = #{escaped.length}"}) do
+      raise e if e
+    end
+  end
+
   def test_cgi_unescapeHTML
     assert_equal("'&\"><", CGI.unescapeHTML("&#39;&amp;&quot;&gt;&lt;"))
   end