8279356: Method linking fails with guarantee(mh->adapter() != NULL) f…

…ailed: Adapter blob must already exist!

Reviewed-by: phh
Backport-of: 6d7db4b0b3e9172645cef12c36fbeb41a6d38d83

src/hotspot/share/oops/method.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -1086,7 +1086,7 @@ void Method::link_method(const methodHandle& h_method, TRAPS) {
     }
     assert(entry == _from_interpreted_entry,
            "should be correctly set during dump time");
-  } else if (_i2i_entry != NULL) {
+  } else if (adapter() != NULL) {
     return;
   }
   assert( _code == NULL, "nothing compiled yet" );

test/hotspot/jtreg/compiler/codecache/OverflowCodeCacheTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014, 2016, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,7 +23,7 @@
 
 /*
  * @test OverflowCodeCacheTest
- * @bug 8059550
+ * @bug 8059550 8279356
  * @summary testing of code cache segments overflow
  * @library /test/lib
  * @modules java.base/jdk.internal.misc
@@ -34,11 +34,14 @@
  *                                sun.hotspot.WhiteBox$WhiteBoxPermission
  * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions
  *                   -XX:+WhiteBoxAPI -XX:CompileCommand=compileonly,null::*
- *                   -XX:-SegmentedCodeCache
- *                   compiler.codecache.OverflowCodeCacheTest
+ *                   -XX:-SegmentedCodeCache -Xmixed
+ *                   compiler.codecache.OverflowCodeCacheTest CompilationDisabled
  * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions
  *                   -XX:+WhiteBoxAPI -XX:CompileCommand=compileonly,null::*
- *                   -XX:+SegmentedCodeCache
+ *                   -XX:+SegmentedCodeCache -Xmixed
+ *                   compiler.codecache.OverflowCodeCacheTest CompilationDisabled
+ * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions
+ *                   -XX:+WhiteBoxAPI -XX:-SegmentedCodeCache -Xmixed
  *                   compiler.codecache.OverflowCodeCacheTest
  */
 
@@ -50,13 +53,21 @@
 import sun.hotspot.code.CodeBlob;
 
 import java.lang.management.MemoryPoolMXBean;
+import java.lang.reflect.Method;
 import java.util.ArrayList;
 import java.util.EnumSet;
 
+class Helper {
+    // Uncommon signature to prevent sharing and force creation of a new adapter
+    public void method(float a, float b, float c, Object o) { }
+}
+
 public class OverflowCodeCacheTest {
     private static final WhiteBox WHITE_BOX = WhiteBox.getWhiteBox();
+    private static boolean COMPILATION_DISABLED = false;
 
     public static void main(String[] args) {
+        COMPILATION_DISABLED = args.length > 0;
         EnumSet<BlobType> blobTypes = BlobType.getAvailable();
         for (BlobType type : blobTypes) {
             new OverflowCodeCacheTest(type).test();
@@ -75,6 +86,8 @@ private void test() {
         System.out.println("allocating till possible...");
         ArrayList<Long> blobs = new ArrayList<>();
         int compilationActivityMode = -1;
+        // Lock compilation to be able to better control code cache space
+        WHITE_BOX.lockCompilation();
         try {
             long addr;
             int size = (int) (getHeapSize() >> 7);
@@ -89,15 +102,43 @@ private void test() {
                 }
             }
             /* now, remember compilationActivityMode to check it later, after freeing, since we
-               possibly have no free cache for futher work */
+               possibly have no free cache for further work */
             compilationActivityMode = WHITE_BOX.getCompilationActivityMode();
+
+            // Use smallest allocation size to make sure all of the available space
+            // is filled up. Don't free these below to put some pressure on the sweeper.
+            while ((addr = WHITE_BOX.allocateCodeBlob(1, type.id)) != 0) { }
         } finally {
+            try {
+                // Trigger creation of a new adapter for Helper::method
+                // which will fail because we are out of code cache space.
+                Helper helper = new Helper();
+            } catch (VirtualMachineError e) {
+                // Expected
+            }
+            // Free code cache space
             for (Long blob : blobs) {
                 WHITE_BOX.freeCodeBlob(blob);
             }
+
+            // Convert some nmethods to zombie and then free them to re-enable compilation
+            WHITE_BOX.unlockCompilation();
+            WHITE_BOX.forceNMethodSweep();
+            WHITE_BOX.forceNMethodSweep();
+
+            // Trigger compilation of Helper::method which will hit an assert because
+            // adapter creation failed above due to a lack of code cache space.
+            Helper helper = new Helper();
+            for (int i = 0; i < 100_000; i++) {
+                helper.method(0, 0, 0, null);
+            }
+        }
+        // Only check this if compilation is disabled, otherwise the sweeper might have
+        // freed enough nmethods to allow for re-enabling compilation.
+        if (COMPILATION_DISABLED) {
+            Asserts.assertNotEquals(compilationActivityMode, 1 /* run_compilation*/,
+                    "Compilation must be disabled when CodeCache(CodeHeap) overflows");
         }
-        Asserts.assertNotEquals(compilationActivityMode, 1 /* run_compilation*/,
-                "Compilation must be disabled when CodeCache(CodeHeap) overflows");
     }
 
     private long getHeapSize() {