8294593: Check the size of the target on invocations of BigInteger::isProbablePrime

Reviewed-by: darcy

Author: rgiulietti

Diff for: src/java.base/share/classes/java/math/BigInteger.java

@@ -3889,7 +3889,9 @@ public boolean isProbablePrime(int certainty) {
             return true;
         if (!w.testBit(0) || w.equals(ONE))
             return false;
-
+        if (w.bitLength() > PRIME_SEARCH_BIT_LENGTH_LIMIT + 1) {
+            throw new ArithmeticException("Primality test implementation restriction on bitLength");
+        }
         return w.primeToCertainty(certainty, null);
     }
 

Diff for: test/jdk/java/math/BigInteger/PrimeTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,7 +26,7 @@
  * @library /test/lib
  * @build jdk.test.lib.RandomFactory
  * @run main PrimeTest
- * @bug 8026236 8074460 8078672
+ * @bug 8026236 8074460 8078672 8294593
  * @summary test primality verification methods in BigInteger (use -Dseed=X to set PRNG seed)
  * @author bpb
  * @key randomness
@@ -86,6 +86,10 @@ public static void main(String[] args) throws Exception {
             throw new Exception("PrimeTest FAILED!");
         }
 
+        if (!checkHugeFails()) {
+            throw new Exception("Primality test on huge integer should fail but succeeded");
+        }
+
         System.out.println("PrimeTest succeeded!");
     }
 
@@ -230,4 +234,19 @@ private static boolean checkMersennePrimes(int certainty) {
 
         return result;
     }
+
+    private static boolean checkHugeFails() {
+        try {
+            // huge odd integer
+            BigInteger a = BigInteger.ONE.shiftLeft(500_000_000 + 1)
+                    .setBit(0);
+            a.isProbablePrime(1);
+            // not expected to reach here
+            return false;
+        } catch (ArithmeticException e) {
+            // this is the expected behavior
+            return true;
+        }
+    }
+
 }