8336829: [lworld] serviceability/jvmti/RedefineClasses/RedefineLeakThrowable.java gets Metaspace OOM

Reviewed-by: fparain

Author: coleenp

Diff for: src/hotspot/share/oops/inlineKlass.cpp

@@ -318,9 +318,11 @@ void InlineKlass::initialize_calling_convention(TRAPS) {
 void InlineKlass::deallocate_contents(ClassLoaderData* loader_data) {
   if (extended_sig() != nullptr) {
     MetadataFactory::free_array<SigEntry>(loader_data, extended_sig());
+    *((Array<SigEntry>**)adr_extended_sig()) = nullptr;
   }
   if (return_regs() != nullptr) {
     MetadataFactory::free_array<VMRegPair>(loader_data, return_regs());
+    *((Array<VMRegPair>**)adr_return_regs()) = nullptr;
   }
   cleanup_blobs();
   InstanceKlass::deallocate_contents(loader_data);

Diff for: src/hotspot/share/oops/instanceKlass.cpp

@@ -720,11 +720,12 @@ void InstanceKlass::deallocate_contents(ClassLoaderData* loader_data) {
 
   if (inline_type_field_klasses_array() != nullptr) {
     MetadataFactory::free_array<InlineKlass*>(loader_data, inline_type_field_klasses_array());
+    set_inline_type_field_klasses_array(nullptr);
   }
-  set_inline_type_field_klasses_array(nullptr);
 
   if (null_marker_offsets_array() != nullptr) {
     MetadataFactory::free_array<int>(loader_data, null_marker_offsets_array());
+    set_null_marker_offsets_array(nullptr);
   }
 
   // If a method from a redefined class is using this constant pool, don't
@@ -766,6 +767,7 @@ void InstanceKlass::deallocate_contents(ClassLoaderData* loader_data) {
       !loadable_descriptors()->is_shared()) {
     MetadataFactory::free_array<jushort>(loader_data, loadable_descriptors());
   }
+  set_loadable_descriptors(nullptr);
 
   // We should deallocate the Annotations instance if it's not in shared spaces.
   if (annotations() != nullptr && !annotations()->is_shared()) {

Diff for: test/hotspot/jtreg/serviceability/jvmti/RedefineClasses/RedefineLeakThrowable.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,7 +26,6 @@
  * @bug 8308762
  * @library /test/lib
  * @summary Test that redefinition of class containing Throwable refs does not leak constant pool
- * @requires os.family == "aix"
  * @requires vm.jvmti
  * @requires vm.flagless
  * @modules java.base/jdk.internal.misc
@@ -36,20 +35,8 @@
  * @run main/othervm/timeout=6000 -javaagent:redefineagent.jar -XX:MetaspaceSize=23m -XX:MaxMetaspaceSize=23m RedefineLeakThrowable
  */
 
-/*
- * @test
- * @bug 8308762
- * @library /test/lib
- * @summary Test that redefinition of class containing Throwable refs does not leak constant pool
- * @requires os.family != "aix"
- * @requires vm.jvmti
- * @requires vm.flagless
- * @modules java.base/jdk.internal.misc
- * @modules java.instrument
- *          java.compiler
- * @run main RedefineClassHelper
- * @run main/othervm/timeout=6000 -javaagent:redefineagent.jar -XX:MetaspaceSize=17m -XX:MaxMetaspaceSize=17m RedefineLeakThrowable
- */
+
+// MaxMetaspaceSize=23m allows InMemoryJavaCompiler to load even if CDS is off.
 
 class Tester {
     void test() {

Diff for: test/hotspot/jtreg/serviceability/jvmti/RedefineClasses/RedefineLeakThrowableValue.java

@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8336829
+ * @library /test/lib
+ * @summary Test that redefinition of a value class containing Throwable refs does not leak constant pool
+ * @enablePreview
+ * @modules java.base/jdk.internal.misc
+ * @modules java.instrument
+ *          java.compiler
+ * @run main RedefineClassHelper
+ * @run main/othervm/timeout=6000 --enable-preview -Xlog:class+load,gc+metaspace+freelist+oom:rt.log -javaagent:redefineagent.jar -XX:MetaspaceSize=23m -XX:MaxMetaspaceSize=23m RedefineLeakThrowableValue
+ */
+
+import jdk.test.lib.compiler.InMemoryJavaCompiler;
+
+value class Tester {
+    void test() {
+        try {
+            int i = 42;
+        } catch (Throwable t) {
+            t.printStackTrace();
+        }
+    }
+}
+
+public class RedefineLeakThrowableValue {
+
+    static final String NEW_TESTER =
+        "value class Tester {" +
+        "   void test() {" +
+        "        try {" +
+        "            int i = 42;" +
+        "        } catch (Throwable t) {" +
+        "            t.printStackTrace();" +
+        "        }" +
+        "    }" +
+        "}";
+
+
+    public static void main(String argv[]) throws Exception {
+        String java_version = System.getProperty("java.specification.version");
+        // Load InMemoryJavaCompiler outside the redefinition loop to prevent random lambdas that load and fill up metaspace.
+        byte[] bytecodes = InMemoryJavaCompiler.compile("Tester", NEW_TESTER,
+                                                        "-source", java_version, "--enable-preview",
+                                                        "--add-exports", "java.base/jdk.internal.vm.annotation=ALL-UNNAMED");
+        for (int i = 0; i < 700; i++) {
+            RedefineClassHelper.redefineClass(Tester.class, bytecodes);
+        }
+    }
+}