8232081: Try to link all classes during dynamic CDS dump

During CDS dynamic dump, link all classes loaded by the builtin class loaders in JVM_BeforeHalt() and JavaThread::invoke_shutdown_hooks().

Reviewed-by: iklam, dholmes

src/hotspot/share/memory/metaspaceShared.cpp

@@ -1702,13 +1702,22 @@ class LinkSharedClassesClosure : public KlassClosure {
   void do_klass(Klass* k) {
     if (k->is_instance_klass()) {
       InstanceKlass* ik = InstanceKlass::cast(k);
-      // Link the class to cause the bytecodes to be rewritten and the
-      // cpcache to be created. Class verification is done according
-      // to -Xverify setting.
-      _made_progress |= MetaspaceShared::try_link_class(ik, THREAD);
-      guarantee(!HAS_PENDING_EXCEPTION, "exception in link_class");
+      // For dynamic CDS dump, only link classes loaded by the builtin class loaders.
+      bool do_linking = DumpSharedSpaces ? true : !ik->is_shared_unregistered_class();
+      if (do_linking) {
+        // Link the class to cause the bytecodes to be rewritten and the
+        // cpcache to be created. Class verification is done according
+        // to -Xverify setting.
+        _made_progress |= MetaspaceShared::try_link_class(ik, THREAD);
+        guarantee(!HAS_PENDING_EXCEPTION, "exception in link_class");
 
-      ik->constants()->resolve_class_constants(THREAD);
+        if (DumpSharedSpaces) {
+          // The following function is used to resolve all Strings in the statically
+          // dumped classes to archive all the Strings. The archive heap is not supported
+          // for the dynamic archive.
+          ik->constants()->resolve_class_constants(THREAD);
+        }
+      }
     }
   }
 };
@@ -1848,7 +1857,7 @@ int MetaspaceShared::preload_classes(const char* class_list_path, TRAPS) {
 
 // Returns true if the class's status has changed
 bool MetaspaceShared::try_link_class(InstanceKlass* ik, TRAPS) {
-  assert(DumpSharedSpaces, "should only be called during dumping");
+  Arguments::assert_is_dumping_archive();
   if (ik->init_state() < InstanceKlass::linked &&
       !SystemDictionaryShared::has_class_failed_verification(ik)) {
     bool saved = BytecodeVerificationLocal;

src/hotspot/share/oops/instanceKlass.hpp

@@ -1264,12 +1264,6 @@ class InstanceKlass: public Klass {
  public:
   u2 idnum_allocated_count() const      { return _idnum_allocated_count; }
 
-public:
-  void set_in_error_state() {
-    assert(DumpSharedSpaces, "only call this when dumping archive");
-    _init_state = initialization_error;
-  }
-
 private:
   // initialization state
   void set_init_state(ClassState state);

src/hotspot/share/prims/jvm.cpp

@@ -469,6 +469,10 @@ extern volatile jint vm_created;
 
 JVM_ENTRY_NO_ENV(void, JVM_BeforeHalt())
   JVMWrapper("JVM_BeforeHalt");
+  // Link all classes for dynamic CDS dumping before vm exit.
+  if (DynamicDumpSharedSpaces) {
+    MetaspaceShared::link_and_cleanup_shared_classes(THREAD);
+  }
   EventShutdown event;
   if (event.should_commit()) {
     event.set_reason("Shutdown requested from Java");

src/hotspot/share/runtime/thread.cpp

@@ -4326,6 +4326,13 @@ void Threads::create_vm_init_libraries() {
 void JavaThread::invoke_shutdown_hooks() {
   HandleMark hm(this);
 
+  // Link all classes for dynamic CDS dumping before vm exit.
+  // Same operation is being done in JVM_BeforeHalt for handling the
+  // case where the application calls System.exit().
+  if (DynamicDumpSharedSpaces) {
+    MetaspaceShared::link_and_cleanup_shared_classes(this);
+  }
+
   // We could get here with a pending exception, if so clear it now.
   if (this->has_pending_exception()) {
     this->clear_pending_exception();

test/hotspot/jtreg/runtime/cds/appcds/dynamicArchive/LinkClassTest.java

@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+/*
+ * @test
+ * @summary Classes loaded by the builtin class loaders should be linked
+ *          during dynamic CDS dump time.
+ * @requires vm.cds
+ * @library /test/lib /test/hotspot/jtreg/runtime/cds/appcds
+ *          /test/hotspot/jtreg/runtime/cds/appcds/dynamicArchive/test-classes
+ * @build LinkClassApp
+ * @run driver ClassFileInstaller -jar link_class_app.jar LinkClassApp Parent Child Parent2 Child2 MyShutdown
+ * @run driver LinkClassTest
+ */
+
+public class LinkClassTest extends DynamicArchiveTestBase {
+    public static void main(String[] args) throws Exception {
+        runTest(LinkClassTest::test);
+    }
+
+    static void test() throws Exception {
+        String topArchiveName = getNewArchiveName();
+        String appJar = ClassFileInstaller.getJarPath("link_class_app.jar");
+        String mainClass = "LinkClassApp";
+
+        // dump archive with the app without calling System.exit().
+        dump(topArchiveName,
+            "-Xlog:class+load,cds+dynamic=info,cds",
+            "-cp", appJar, mainClass)
+            .assertNormalExit(output -> {
+                output.shouldNotContain("Skipping Parent: Not linked")
+                      .shouldNotContain("Skipping Parent2: Not linked")
+                      .shouldNotContain("Skipping Child: Not linked")
+                      .shouldNotContain("Skipping Child2: Not linked")
+                      .shouldHaveExitValue(0);
+            });
+
+        run(topArchiveName,
+            "-Xlog:class+load",
+            "-cp", appJar, mainClass, "run")
+            .assertNormalExit(output -> {
+                output.shouldContain("Parent source: shared objects file (top)")
+                      .shouldContain("Parent2 source: shared objects file (top)")
+                      .shouldContain("Child source: shared objects file (top)")
+                      .shouldContain("Child2 source: shared objects file (top)")
+                      .shouldHaveExitValue(0);
+            });
+
+        // dump archive with the app calling System.exit().
+        dump(topArchiveName,
+            "-Xlog:class+load,cds+dynamic=info,cds",
+            "-cp", appJar, mainClass, "callExit")
+            .assertNormalExit(output -> {
+                output.shouldNotContain("Skipping Parent: Not linked")
+                      .shouldNotContain("Skipping Parent2: Not linked")
+                      .shouldNotContain("Skipping Child: Not linked")
+                      .shouldNotContain("Skipping Child2: Not linked")
+                      .shouldHaveExitValue(0);
+            });
+
+        run(topArchiveName,
+            "-Xlog:class+load",
+            "-cp", appJar, mainClass, "run")
+            .assertNormalExit(output -> {
+                output.shouldContain("Parent source: shared objects file (top)")
+                      .shouldContain("Parent2 source: shared objects file (top)")
+                      .shouldContain("Child source: shared objects file (top)")
+                      .shouldContain("Child2 source: shared objects file (top)")
+                      .shouldHaveExitValue(0);
+            });
+    }
+}

test/hotspot/jtreg/runtime/cds/appcds/dynamicArchive/test-classes/LinkClassApp.java

@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+class Parent {
+    int get() {return 1;}
+}
+
+class Child extends Parent {
+    int get() {return 2;}
+    int dummy() {
+        // When Child is linked during dynamic dump (when the VM is shutting
+        // down), it will be verified, which will recursively cause Child2/Parent2
+        // to be loaded and verified.
+        // We want to make sure that this is done at a point in the VM lifecycle where
+        // it's still safe to do so.
+        Parent2 x = new Child2();
+        return x.get();
+    }
+}
+
+class Parent2 {
+    int get() {return 3;}
+}
+
+class Child2 extends Parent2 {
+    int get() {return 4;}
+}
+
+class MyShutdown extends Thread{
+    public void run(){
+        System.out.println("shut down hook invoked...");
+    }
+}
+
+class LinkClassApp {
+    public static void main(String args[]) {
+        Runtime r=Runtime.getRuntime();
+        r.addShutdownHook(new MyShutdown());
+
+        if (args.length > 0 && args[0].equals("run")) {
+            System.out.println("test() = " + test());
+        } else {
+            // Executed during dynamic dumping.
+            System.out.println("Test.class is initialized.");
+            System.out.println("Parent.class and Child.class are loaded when Test.class is verified,");
+            System.out.println("but these two classes are not linked");
+        }
+
+        if (args.length > 0 && args[0].equals("callExit")) {
+            System.exit(0);
+        }
+    }
+
+    static int test() {
+        // Verification of Test.test() would load Child and Parent, and create a verification constraint that
+        // Child must be a subtype of Parent.
+        //
+        // Child and Parent are not linked until Test.test() is actually executed.
+        Parent x = new Child();
+        return x.get();
+    }
+}