[Bug #19323] Raise RangeError instead of integer overflow

bignum.c

@@ -4587,11 +4587,14 @@ big_shift3(VALUE x, int lshift_p, size_t shift_numdigits, int shift_numbits)
 
     if (lshift_p) {
         if (LONG_MAX < shift_numdigits) {
-            rb_raise(rb_eArgError, "too big number");
+          too_big:
+            rb_raise(rb_eRangeError, "shift width too big");
         }
         s1 = shift_numdigits;
         s2 = shift_numbits;
+        if ((size_t)s1 != shift_numdigits) goto too_big;
         xn = BIGNUM_LEN(x);
+        if (LONG_MAX/SIZEOF_BDIGIT <= xn+s1) goto too_big;
         z = bignew(xn+s1+1, BIGNUM_SIGN(x));
         zds = BDIGITS(z);
         BDIGITS_ZERO(zds, s1);

test/ruby/test_integer.rb

@@ -2,16 +2,9 @@
 require 'test/unit'
 
 class TestInteger < Test::Unit::TestCase
-  BDSIZE = 0x4000000000000000.coerce(0)[0].size
-  def self.bdsize(x)
-    ((x + 1) / 8 + BDSIZE) / BDSIZE * BDSIZE
-  end
-  def bdsize(x)
-    self.class.bdsize(x)
-  end
-
   FIXNUM_MIN = RbConfig::LIMITS['FIXNUM_MIN']
   FIXNUM_MAX = RbConfig::LIMITS['FIXNUM_MAX']
+  LONG_MAX = RbConfig::LIMITS['LONG_MAX']
 
   def test_aref
 
@@ -96,11 +89,16 @@ def test_lshift
     assert_equal(0, 1 << -0x40000001)
     assert_equal(0, 1 << -0x80000000)
     assert_equal(0, 1 << -0x80000001)
-    # assert_equal(bdsize(0x80000000), (1 << 0x80000000).size)
+
+    char_bit = RbConfig::LIMITS["UCHAR_MAX"].bit_length
+    size_max = RbConfig::LIMITS["SIZE_MAX"]
+    size_bit_max = size_max * char_bit
+    assert_raise_with_message(RangeError, /shift width/) {
+      1 << size_bit_max
+    }
   end
 
   def test_rshift
-    # assert_equal(bdsize(0x40000001), (1 >> -0x40000001).size)
     assert_predicate((1 >> 0x80000000), :zero?)
     assert_predicate((1 >> 0xffffffff), :zero?)
     assert_predicate((1 >> 0x100000000), :zero?)