Raise RangeError for out of bounds unpack_at

This corrects the error raised for the case where the unpack spec
requests a size that's outside the string, which relates to
CVE-2018-8778.

The other two cases that raised the ArgumentError for "outside of
string" appear to be correct, since making them raise RangeError
causes spec failures.

This addresses a security spec failure as shown in jruby#6304.

core/src/main/java/org/jruby/util/Pack.java

@@ -1507,16 +1507,25 @@ private static IRubyObject unpack_A(ThreadContext context, Block block, RubyArra
 
     private static void unpack_at(Ruby runtime, ByteList encodedString, ByteBuffer encode, int occurrences) {
         try {
+            int limit;
             if (occurrences == IS_STAR) {
-                positionBuffer(encode, encodedString.begin() + encode.remaining());
+                limit = checkLimit(runtime, encode, encodedString.begin() + encode.remaining());
             } else {
-                positionBuffer(encode, encodedString.begin() + occurrences);
+                limit = checkLimit(runtime, encode, encodedString.begin() + occurrences);
             }
+            positionBuffer(encode, limit);
         } catch (IllegalArgumentException iae) {
             throw runtime.newArgumentError("@ outside of string");
         }
     }
 
+    private static int checkLimit(Ruby runtime, ByteBuffer encode, int limit) {
+        if (limit >= encode.capacity() || limit < 0) {
+            throw runtime.newRangeError("pack length too big");
+        }
+        return limit;
+    }
+
     @Deprecated
     public static RubyArray unpackWithBlock(ThreadContext context, Ruby runtime, ByteList encodedString, ByteList formatString, Block block) {
         return unpackWithBlock(context, RubyString.newStringLight(runtime, encodedString), formatString, block);