8285914: AppCDS crash when using shared archive with old class file

Reviewed-by: dholmes, iklam

src/hotspot/share/classfile/systemDictionaryShared.cpp

@@ -1625,17 +1625,16 @@ class CleanupDumpTimeLambdaProxyClassTable: StackObj {
   bool do_entry(LambdaProxyClassKey& key, DumpTimeLambdaProxyClassInfo& info) {
     assert_lock_strong(DumpTimeTable_lock);
     InstanceKlass* caller_ik = key.caller_ik();
-    if (SystemDictionaryShared::check_for_exclusion(caller_ik, NULL)) {
-      // If the caller class is excluded, unregister all the associated lambda proxy classes
-      // so that they will not be included in the CDS archive.
-      for (int i = info._proxy_klasses->length() - 1; i >= 0; i--) {
-        SystemDictionaryShared::reset_registered_lambda_proxy_class(info._proxy_klasses->at(i));
-        info._proxy_klasses->remove_at(i);
-      }
-    }
+    InstanceKlass* nest_host = caller_ik->nest_host_not_null();
+
+    // If the caller class and/or nest_host are excluded, the associated lambda proxy
+    // must also be excluded.
+    bool always_exclude = SystemDictionaryShared::check_for_exclusion(caller_ik, NULL) ||
+                          SystemDictionaryShared::check_for_exclusion(nest_host, NULL);
+
     for (int i = info._proxy_klasses->length() - 1; i >= 0; i--) {
       InstanceKlass* ik = info._proxy_klasses->at(i);
-      if (SystemDictionaryShared::check_for_exclusion(ik, NULL)) {
+      if (always_exclude || SystemDictionaryShared::check_for_exclusion(ik, NULL)) {
         SystemDictionaryShared::reset_registered_lambda_proxy_class(ik);
         info._proxy_klasses->remove_at(i);
       }

src/hotspot/share/oops/instanceKlass.hpp

@@ -474,6 +474,11 @@ class InstanceKlass: public Klass {
   bool has_nest_member(JavaThread* current, InstanceKlass* k) const;
 
 public:
+  // Call this only if you know that the nest host has been initialized.
+  InstanceKlass* nest_host_not_null() {
+    assert(_nest_host != NULL, "must be");
+    return _nest_host;
+  }
   // Used to construct informative IllegalAccessError messages at a higher level,
   // if there was an issue resolving or validating the nest host.
   // Returns NULL if there was no error.

test/hotspot/jtreg/runtime/cds/appcds/NestHostOldInf.java

@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+/*
+ * @test
+ * @bug 8285914
+ * @summary A lambda proxy class should not be archived if its nest host implements an
+ *          old (with major version < JDK_6 (50)) interface which cannot be verified during dump time.
+ * @requires vm.cds
+ * @library /test/lib
+ * @compile test-classes/OldInf.jasm test-classes/ChildOldInf.java test-classes/NestHostOldInfApp.java
+ * @run driver NestHostOldInf
+ */
+
+import jdk.test.lib.cds.CDSTestUtils;
+import jdk.test.lib.process.OutputAnalyzer;
+
+public class NestHostOldInf {
+
+    public static void main(String[] args) throws Exception {
+        String mainClass = "NestHostOldInfApp";
+        String namePrefix = "oldsuperinf";
+        String appClasses[] = TestCommon.list("OldInf", "ChildOldInf", "ChildOldInf$InnerChild", mainClass);
+        JarBuilder.build(namePrefix, appClasses);
+
+        String appJar = TestCommon.getTestJar(namePrefix + ".jar");
+        String archiveName = namePrefix + ".jsa";
+
+        boolean dynamicMode = CDSTestUtils.DYNAMIC_DUMP;
+
+        // create archive with class list
+        OutputAnalyzer output = TestCommon.dump(appJar, appClasses, "-Xlog:class+load,cds=debug,verification=trace");
+        TestCommon.checkExecReturn(output, 0,
+                                   dynamicMode ? true : false,
+                                   "Skipping OldInf: Old class has been linked",
+                                   "Skipping ChildOldInf: Old class has been linked");
+
+        // run with archive
+        TestCommon.run(
+            "-cp", appJar,
+            "-Xlog:class+load,cds=debug,verification=trace",
+            mainClass)
+          .assertNormalExit(out -> {
+              out.shouldContain("Verifying class OldInf with old format")
+                 .shouldContain("Verifying class ChildOldInf with new format")
+                 .shouldContain("ChildOldInf$InnerChild source: shared objects file")
+                 .shouldContain("Hello from InnerChild doTest");
+              if (!dynamicMode) {
+                  out.shouldContain("OldInf source: shared objects file")
+                     .shouldContain("ChildOldInf source: shared objects file");
+              } else {
+                  // Old classes were already linked before dynamic dump happened,
+                  // so they couldn't be archived.
+                  out.shouldMatch(".class.load.*OldInf source:.*oldsuperinf.jar")
+                     .shouldMatch(".class.load.*ChildOldInf source:.*oldsuperinf.jar");
+              }
+          });
+    }
+}

test/hotspot/jtreg/runtime/cds/appcds/dynamicArchive/NestHostOldInf.java

@@ -0,0 +1,89 @@
+/*
+ * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+/*
+ * @test
+ * @bug 8285914
+ * @summary A lambda proxy class should not be archived if its nest host implements an
+ *          old (with major version < JDK_6 (50)) interface which cannot be verified during dump time.
+ * @requires vm.cds
+ * @requires vm.cds.custom.loaders
+ * @library /test/lib /test/hotspot/jtreg/runtime/cds/appcds
+ * @compile ../test-classes/OldInf.jasm ../test-classes/ChildOldInf.java ../test-classes/NestHostOldInfApp.java
+ * @build sun.hotspot.WhiteBox
+ * @run driver jdk.test.lib.helpers.ClassFileInstaller -jar oldclassapp.jar NestHostOldInfApp OldInf ChildOldInf ChildOldInf$InnerChild
+ * @run driver jdk.test.lib.helpers.ClassFileInstaller -jar WhiteBox.jar sun.hotspot.WhiteBox
+ * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -Xbootclasspath/a:./WhiteBox.jar NestHostOldInf
+ */
+
+import java.io.File;
+import jdk.test.lib.cds.CDSTestUtils;
+import jdk.test.lib.process.OutputAnalyzer;
+import jdk.test.lib.helpers.ClassFileInstaller;
+
+public class NestHostOldInf extends DynamicArchiveTestBase {
+    private static final String ARCHIVE_NAME = CDSTestUtils.getOutputFileName("oldclass-top.jsa");
+    private static String wbJar = ClassFileInstaller.getJarPath("WhiteBox.jar");
+    private static String use_whitebox_jar = "-Xbootclasspath/a:" + wbJar;
+    private static String appJar = ClassFileInstaller.getJarPath("oldclassapp.jar");
+    private static String mainAppClass = "NestHostOldInfApp";
+
+    public static void main(String[] args) throws Exception {
+        runTest(NestHostOldInf::doTest);
+    }
+
+    private static void doTest() throws Exception {
+        dump(ARCHIVE_NAME,
+             use_whitebox_jar,
+             "-XX:+UnlockDiagnosticVMOptions",
+             "-XX:+WhiteBoxAPI",
+             "-Xlog:cds",
+             "-Xlog:cds+dynamic=debug",
+             "-cp", appJar,
+             mainAppClass)
+             .assertNormalExit(output -> {
+                 output.shouldContain("Written dynamic archive 0x")
+                       .shouldContain("Skipping ChildOldInf: Old class has been linked")
+                       .shouldContain("Skipping OldInf: Old class has been linked")
+                       .shouldHaveExitValue(0);
+                 });
+
+        run(ARCHIVE_NAME,
+            use_whitebox_jar,
+            "-XX:+UnlockDiagnosticVMOptions",
+            "-XX:+WhiteBoxAPI",
+            "-Xlog:class+load",
+            "-Xlog:cds=debug",
+            "-Xlog:cds+dynamic=info",
+            "-cp", appJar,
+            mainAppClass)
+            .assertNormalExit(output -> {
+                output.shouldHaveExitValue(0)
+                      .shouldMatch(".class.load. OldInf source:.*oldclassapp.jar")
+                      .shouldMatch(".class.load. ChildOldInf source:.*oldclassapp.jar")
+                      .shouldContain("ChildOldInf$InnerChild source: shared objects file (top)")
+                      .shouldMatch(".class.load. ChildOldInf[$]InnerChild[$][$]Lambda[$].*/0x.*source:.ChildOldInf");
+                });
+    }
+}

test/hotspot/jtreg/runtime/cds/appcds/test-classes/ChildOldInf.java

@@ -25,4 +25,21 @@ public class ChildOldInf implements OldInf {
     public String doit() {
         return "ChildOldInf";
     }
+
+    public static ChildOldInf.InnerChild innerChild() {
+        return new InnerChild();
+    }
+
+    public static final class InnerChild {
+        public InnerChild() {
+        }
+        public void doTest() {
+            doit(() -> {
+                System.out.println("Hello from InnerChild doTest");
+            });
+        }
+        public void doit(Runnable t) {
+            t.run();
+        }
+    }
 }

test/hotspot/jtreg/runtime/cds/appcds/test-classes/NestHostOldInfApp.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+public class NestHostOldInfApp {
+    public static void main(String args[]) {
+        ChildOldInf.innerChild().doTest();
+    }
+}