8261860: Crash caused by lambda proxy class loaded in Shutdown hook

Reviewed-by: iklam, minqi

src/hotspot/share/classfile/systemDictionaryShared.cpp

@@ -1600,7 +1600,9 @@ void SystemDictionaryShared::add_lambda_proxy_class(InstanceKlass* caller_ik,
   InstanceKlass* nest_host = caller_ik->nest_host(THREAD);
 
   DumpTimeSharedClassInfo* info = _dumptime_table->get(lambda_ik);
-  if (info != NULL && !lambda_ik->is_non_strong_hidden() && is_builtin(lambda_ik) && is_builtin(caller_ik)) {
+  if (info != NULL && !lambda_ik->is_non_strong_hidden() && is_builtin(lambda_ik) && is_builtin(caller_ik)
+      // Don't include the lambda proxy if its nest host is not in the "linked" state.
+      && nest_host->is_linked()) {
     // Set _is_archived_lambda_proxy in DumpTimeSharedClassInfo so that the lambda_ik
     // won't be excluded during dumping of shared archive. See ExcludeDumpTimeSharedClasses.
     info->_is_archived_lambda_proxy = true;

test/hotspot/jtreg/runtime/cds/appcds/dynamicArchive/LambdaProxyDuringShutdown.java

@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+/*
+ * @test
+ * @bug 8261860
+ * @summary VM should not crash if a lambda proxy class is created during
+ *          shutdown and its nest host is not linked.
+ * @requires vm.cds
+ * @library /test/lib /test/hotspot/jtreg/runtime/cds/appcds
+ *          /test/hotspot/jtreg/runtime/cds/appcds/dynamicArchive/test-classes
+ * @build LambdaProxyDuringShutdownApp sun.hotspot.WhiteBox LambdaVerification
+ * @run driver ClassFileInstaller -jar lambda_proxy_shutdown.jar LambdaVerification
+ *             LambdaProxyDuringShutdownApp MyShutdown Outer Outer$Inner
+ * @run driver ClassFileInstaller -jar WhiteBox.jar sun.hotspot.WhiteBox
+ * @run driver ClassFileInstaller sun.hotspot.WhiteBox
+ * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -Xbootclasspath/a:. LambdaProxyDuringShutdown
+ */
+
+public class LambdaProxyDuringShutdown extends DynamicArchiveTestBase {
+    public static void main(String[] args) throws Exception {
+        runTest(LambdaProxyDuringShutdown::test);
+    }
+
+    static void test() throws Exception {
+        String topArchiveName = getNewArchiveName();
+        String appJar = ClassFileInstaller.getJarPath("lambda_proxy_shutdown.jar");
+        String mainClass = "LambdaProxyDuringShutdownApp";
+        String wbJar = ClassFileInstaller.getJarPath("WhiteBox.jar");
+        String use_whitebox_jar = "-Xbootclasspath/a:" + wbJar;
+        String appOutput = "Hello from Inner";
+
+        dump(topArchiveName,
+            "-XX:+UnlockDiagnosticVMOptions",
+            "-XX:+WhiteBoxAPI",
+            "-Xlog:class+load=debug,cds=debug,cds+dynamic=info",
+            use_whitebox_jar,
+            "-cp", appJar, mainClass)
+            .assertNormalExit(output -> {
+                // Nest host should be skipped since it is not in the linked state.
+                output.shouldContain("Skipping Outer: Not linked")
+                // Lambda proxy is loaded normally.
+                      .shouldMatch("class.load.*Outer[$]Inner[$][$]Lambda[$].*0x.*source:.Outer")
+                      .shouldContain(appOutput)
+                      .shouldHaveExitValue(0);
+            });
+
+        run(topArchiveName,
+            "-XX:+UnlockDiagnosticVMOptions",
+            "-XX:+WhiteBoxAPI",
+            use_whitebox_jar,
+            "-Xlog:class+load=debug",
+            "-cp", appJar, mainClass, "run")
+            .assertNormalExit(output -> {
+                // Only the Inner class is loaded from the dynamic archive.
+                // The nest host (Outer) and its lambda proxy are not loaded
+                // from the dynamic archive.
+                output.shouldMatch("class.load.*Outer.source:.*lambda_proxy_shutdown.jar")
+                      .shouldMatch("class.load.*Outer[$]Inner[$][$]Lambda[$].*0x.*source:.Outer")
+                      .shouldMatch("class.load. Outer[$]Inner.source:.*shared.*objects.*file.*(top)")
+                      .shouldContain(appOutput)
+                      .shouldHaveExitValue(0);
+            });
+    }
+}

test/hotspot/jtreg/runtime/cds/appcds/dynamicArchive/test-classes/LambdaProxyDuringShutdownApp.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+class Outer{
+    static final class Inner{
+        static {
+            doit(() -> {
+                System.out.println("Hello from Inner");
+            });
+        }
+        static void doit(Runnable t) {
+            t.run();
+        }
+    }
+}
+
+class MyShutdown extends Thread {
+    public void run() {
+        Outer.Inner inner = new Outer.Inner();
+    }
+}
+
+public class LambdaProxyDuringShutdownApp {
+  public static void main(String[] args) throws Exception {
+      Runtime r = Runtime.getRuntime();
+      r.addShutdownHook(new MyShutdown());
+      System.exit(0);
+  }
+}