8295153: java/util/Base64/TestEncodingDecodingLength.java ran out of …

…memory

Reviewed-by: lancea, naoto

test/jdk/java/util/Base64/TestEncodingDecodingLength.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -21,55 +21,95 @@
  * questions.
  */
 
-import java.nio.ByteBuffer;
-import java.util.Arrays;
+import org.junit.jupiter.api.Test;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
 import java.util.Base64;
 
-/**
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.fail;
+
+/*
  * @test
- * @bug 8210583 8217969 8218265
- * @summary Tests Base64.Encoder.encode and Base64.Decoder.decode
- *          with the large size of input array/buffer
- * @requires (sun.arch.data.model == "64" & os.maxMemory >= 10g)
- * @run main/othervm -Xms6g -Xmx8g TestEncodingDecodingLength
- *
+ * @bug 8210583 8217969 8218265 8295153
+ * @summary White-box test that effectively checks Base64.Encoder.encode and
+ *          Base64.Decoder.decode behavior with large, (Integer.MAX_VALUE) sized
+ *          input array/buffer. Tests the private methods "encodedOutLength" and
+ *          "decodedOutLength".
+ * @run junit/othervm --add-opens java.base/java.util=ALL-UNNAMED TestEncodingDecodingLength
  */
 
+// We perform a white-box test due to the heavy memory usage that testing
+// the public API would require which has shown to cause intermittent issues
 public class TestEncodingDecodingLength {
 
-    public static void main(String[] args) {
-        int size = Integer.MAX_VALUE - 8;
-        byte[] inputBytes = new byte[size];
-        byte[] outputBytes = new byte[size];
+    // A value large enough to test the desired memory conditions in encode and decode
+    private static final int LARGE_MEM_SIZE = Integer.MAX_VALUE - 8;
+    private static final Base64.Decoder DECODER = Base64.getDecoder();
+    private static final Base64.Encoder ENCODER = Base64.getEncoder();
 
-        // Check encoder with large array length
-        Base64.Encoder encoder = Base64.getEncoder();
-        checkOOM("encode(byte[])", () -> encoder.encode(inputBytes));
-        checkIAE("encode(byte[] byte[])", () -> encoder.encode(inputBytes, outputBytes));
-        checkOOM("encodeToString(byte[])", () -> encoder.encodeToString(inputBytes));
-        checkOOM("encode(ByteBuffer)", () -> encoder.encode(ByteBuffer.wrap(inputBytes)));
-
-        // Check decoder with large array length,
-        // should not throw any exception
-        Arrays.fill(inputBytes, (byte) 86);
-        Base64.Decoder decoder = Base64.getDecoder();
-        decoder.decode(inputBytes);
-        decoder.decode(inputBytes, outputBytes);
-        decoder.decode(ByteBuffer.wrap(inputBytes));
+    // Effectively tests that encode(byte[] src, byte[] dst) throws an
+    // IllegalArgumentException with array sized near Integer.MAX_VALUE. All the
+    // encode() methods call encodedOutLength(), which is where the OOME is expected
+    @Test
+    public void largeEncodeIAETest() throws IllegalAccessException,
+            InvocationTargetException, NoSuchMethodException {
+        Method m = getMethod(ENCODER,
+                "encodedOutLength", int.class, boolean.class);
+        // When throwOOME param is false, encodedOutLength should return -1 in
+        // this situation, which encode() uses to throw IAE
+        assertEquals(-1, m.invoke(ENCODER, LARGE_MEM_SIZE, false));
     }
 
-    private static final void checkOOM(String methodName, Runnable r) {
+    // Effectively tests that the overloaded encode() and encodeToString() methods
+    // throw OutOfMemoryError with array/buffer sized near Integer.MAX_VALUE
+    @Test
+    public void largeEncodeOOMETest() throws IllegalAccessException, NoSuchMethodException {
+        Method m = getMethod(ENCODER,
+                "encodedOutLength", int.class, boolean.class);
         try {
-            r.run();
-            throw new RuntimeException("OutOfMemoryError should have been thrown by: " + methodName);
-        } catch (OutOfMemoryError er) {}
+            m.invoke(ENCODER, LARGE_MEM_SIZE, true);
+        } catch (InvocationTargetException ex) {
+            Throwable rootEx = ex.getCause();
+            assertEquals(OutOfMemoryError.class, rootEx.getClass(),
+                    "OOME should be thrown with Integer.MAX_VALUE input");
+            assertEquals("Encoded size is too large", rootEx.getMessage());
+        }
     }
 
-    private static final void checkIAE(String methodName, Runnable r) {
+    // Effectively tests that the overloaded decode() methods do not throw
+    // OOME nor NASE with array/buffer sized near Integer.MAX_VALUE All the decode
+    // methods call decodedOutLength(), which is where the previously thrown
+    // OOME or NASE would occur at.
+    @Test
+    public void largeDecodeTest() throws IllegalAccessException, NoSuchMethodException {
+        Method m = getMethod(DECODER,
+                "decodedOutLength", byte[].class, int.class, int.class);
+        byte[] src = {1};
         try {
-            r.run();
-            throw new RuntimeException("IllegalArgumentException should have been thrown by: " + methodName);
-        } catch (IllegalArgumentException iae) {}
+            /*
+             decodedOutLength() takes the src array, position, and limit as params.
+             The src array will be indexed at limit-1 to search for padding.
+             To avoid passing an array with Integer.MAX_VALUE memory allocated, we
+             set position param to be -size. Since the initial length
+             is calculated as limit - position. This mocks the potential overflow
+             calculation and still allows the array to be indexed without an AIOBE.
+            */
+            m.invoke(DECODER, src, -LARGE_MEM_SIZE + 1, 1);
+        } catch (InvocationTargetException ex) {
+            // 8210583 - decode no longer throws NASE
+            // 8217969 - decode no longer throws OOME
+            fail("Decode threw an unexpected exception with " +
+                    "Integer.MAX_VALUE sized input: " + ex.getCause());
+        }
     }
-}
 
+    // Utility to get the private visibility method
+    private static Method getMethod(Object obj, String methodName, Class<?>... params)
+            throws NoSuchMethodException {
+        Method m = obj.getClass().getDeclaredMethod(methodName, params);
+        m.setAccessible(true);
+        return m;
+    }
+}