8292695: SIGQUIT and jcmd attaching mechanism does not work with signal chaining library

8283337: Posix signal handler modification warning triggering incorrectly

Reviewed-by: manc, mdoerr
Backport-of: 45ff10cc68296c7c73d0eafe6fcc9946ab98267e

Author: GoeLin

src/hotspot/os/posix/signals_posix.cpp

@@ -341,7 +341,7 @@ static const struct {
 };
 
 ////////////////////////////////////////////////////////////////////////////////
-// sun.misc.Signal support
+// sun.misc.Signal and BREAK_SIGNAL support
 
 void jdk_misc_signal_init() {
   // Initialize signal structures
@@ -562,11 +562,6 @@ int JVM_HANDLE_XXX_SIGNAL(int sig, siginfo_t* info,
 {
   assert(info != NULL && ucVoid != NULL, "sanity");
 
-  if (sig == BREAK_SIGNAL) {
-    assert(!ReduceSignalUsage, "Should not happen with -Xrs/-XX:+ReduceSignalUsage");
-    return true; // ignore it
-  }
-
   // Note: it's not uncommon that JNI code uses signal/sigset to install,
   // then restore certain signal handler (e.g. to temporarily block SIGPIPE,
   // or have a SIGILL handler when detecting CPU type). When that happens,
@@ -1202,7 +1197,7 @@ int os::get_signal_number(const char* signal_name) {
   return -1;
 }
 
-void set_signal_handler(int sig, bool do_check = true) {
+void set_signal_handler(int sig) {
   // Check for overwrite.
   struct sigaction oldAct;
   sigaction(sig, (struct sigaction*)NULL, &oldAct);
@@ -1244,9 +1239,10 @@ void set_signal_handler(int sig, bool do_check = true) {
   }
 #endif
 
-  // Save handler setup for later checking
+  // Save handler setup for possible later checking
   vm_handlers.set(sig, &sigAct);
-  do_check_signal_periodically[sig] = do_check;
+
+  do_check_signal_periodically[sig] = true;
 
   int ret = sigaction(sig, &sigAct, &oldAct);
   assert(ret == 0, "check");
@@ -1285,11 +1281,24 @@ void install_signal_handlers() {
   PPC64_ONLY(set_signal_handler(SIGTRAP);)
   set_signal_handler(SIGXFSZ);
   if (!ReduceSignalUsage) {
-    // This is just for early initialization phase. Intercepting the signal here reduces the risk
-    // that an attach client accidentally forces HotSpot to quit prematurely. We skip the periodic
-    // check because late initialization will overwrite it to UserHandler.
-    set_signal_handler(BREAK_SIGNAL, false);
+    // Install BREAK_SIGNAL's handler in early initialization phase, in
+    // order to reduce the risk that an attach client accidentally forces
+    // HotSpot to quit prematurely.
+    // The actual work for handling BREAK_SIGNAL is performed by the Signal
+    // Dispatcher thread, which is created and started at a much later point,
+    // see os::initialize_jdk_signal_support(). Any BREAK_SIGNAL received
+    // before the Signal Dispatcher thread is started is queued up via the
+    // pending_signals[BREAK_SIGNAL] counter, and will be processed by the
+    // Signal Dispatcher thread in a delayed fashion.
+    //
+    // Also note that HotSpot does NOT support signal chaining for BREAK_SIGNAL.
+    // Applications that require a custom BREAK_SIGNAL handler should run with
+    // -XX:+ReduceSignalUsage. Otherwise if libjsig is used together with
+    // -XX:+ReduceSignalUsage, libjsig will prevent changing BREAK_SIGNAL's
+    // handler to a custom handler.
+    os::signal(BREAK_SIGNAL, os::user_handler());
   }
+
 #if defined(__APPLE__)
   // lldb (gdb) installs both standard BSD signal handlers, and mach exception
   // handlers. By replacing the existing task exception handler, we disable lldb's mach
@@ -1784,12 +1793,12 @@ int PosixSignals::init() {
 
   signal_sets_init();
 
-  install_signal_handlers();
-
-  // Initialize data for jdk.internal.misc.Signal
+  // Initialize data for jdk.internal.misc.Signal and BREAK_SIGNAL's handler.
   if (!ReduceSignalUsage) {
     jdk_misc_signal_init();
   }
 
+  install_signal_handlers();
+
   return JNI_OK;
 }

src/hotspot/os/windows/os_windows.cpp

@@ -2266,6 +2266,15 @@ static void jdk_misc_signal_init() {
 
   // Add a CTRL-C handler
   SetConsoleCtrlHandler(consoleHandler, TRUE);
+
+  // Initialize sigbreakHandler.
+  // The actual work for handling CTRL-BREAK is performed by the Signal
+  // Dispatcher thread, which is created and started at a much later point,
+  // see os::initialize_jdk_signal_support(). Any CTRL-BREAK received
+  // before the Signal Dispatcher thread is started is queued up via the
+  // pending_signals[SIGBREAK] counter, and will be processed by the
+  // Signal Dispatcher thread in a delayed fashion.
+  os::signal(SIGBREAK, os::user_handler());
 }
 
 void os::signal_notify(int sig) {
@@ -4386,7 +4395,8 @@ jint os::init_2(void) {
 
   SymbolEngine::recalc_search_path();
 
-  // Initialize data for jdk.internal.misc.Signal
+  // Initialize data for jdk.internal.misc.Signal, and install CTRL-C and
+  // CTRL-BREAK handlers.
   if (!ReduceSignalUsage) {
     jdk_misc_signal_init();
   }

src/hotspot/share/runtime/os.cpp

@@ -513,8 +513,6 @@ void os::initialize_jdk_signal_support(TRAPS) {
       Threads::add(signal_thread);
       Thread::start(signal_thread);
     }
-    // Handle ^BREAK
-    os::signal(SIGBREAK, os::user_handler());
   }
 }
 

test/hotspot/jtreg/runtime/Thread/TestBreakSignalThreadDump.java

@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, Google and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test id=default
+ * @bug 8292695
+ * @summary Check that Ctrl-\ or Ctrl-Break (on Windows) causes HotSpot VM to print a full thread dump.
+ * @library /vmTestbase
+ *          /test/lib
+ * @run driver TestBreakSignalThreadDump
+ */
+
+/*
+ * @test id=with_jsig
+ * @bug 8292695
+ * @summary Check that Ctrl-\ causes HotSpot VM to print a full thread dump when signal chaining is used.
+ * @requires os.family != "windows" & os.family != "aix"
+ * @library /vmTestbase
+ *          /test/lib
+ * @run driver TestBreakSignalThreadDump load_libjsig
+ */
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Map;
+import jdk.test.lib.Platform;
+import jdk.test.lib.Utils;
+import jdk.test.lib.process.ProcessTools;
+import jdk.test.lib.process.OutputAnalyzer;
+import vm.share.ProcessUtils;
+
+public class TestBreakSignalThreadDump {
+
+    static class TestProcess {
+        static {
+            System.loadLibrary("ProcessUtils");
+        }
+
+        public static void main(String[] argv) throws Exception {
+            ProcessUtils.sendCtrlBreak();
+            // Wait a bit, as JVM processes the break signal asynchronously.
+            Thread.sleep(1000);
+            System.out.println("Done!");
+        }
+    }
+
+    public static void main(String[] argv) throws Exception {
+        String main = "TestBreakSignalThreadDump$TestProcess";
+        ProcessBuilder pb = ProcessTools.createTestJvm("-Djava.library.path=" + Utils.TEST_NATIVE_PATH, main);
+
+        if (argv.length > 0 && argv[0].equals("load_libjsig")) {
+            prepend_jsig_lib(pb.environment());
+        }
+
+        OutputAnalyzer output = new OutputAnalyzer(pb.start());
+        output.shouldHaveExitValue(0);
+        output.shouldContain("Full thread dump ");
+        output.shouldContain("java.lang.Thread.State: RUNNABLE");
+        output.shouldContain("Done!");
+    }
+
+    private static void prepend_jsig_lib(Map<String, String> env) {
+        Path libjsig = Platform.jvmLibDir().resolve("libjsig." + Platform.sharedLibraryExt());
+        if (!Files.exists(libjsig)) {
+            throw new RuntimeException("File libjsig not found, path: " + libjsig);
+        }
+        String env_var = Platform.isOSX() ? "DYLD_INSERT_LIBRARIES" : "LD_PRELOAD";
+        env.put(env_var, libjsig.toString());
+    }
+}