8279124: VM does not handle SIGQUIT during initialization

Backport-of: 9bf6ffa19f1ea9efcadb3396d921305c9ec0b1d1

Author: Xin Liu

src/hotspot/os/posix/signals_posix.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -562,6 +562,11 @@ int JVM_HANDLE_XXX_SIGNAL(int sig, siginfo_t* info,
 {
   assert(info != NULL && ucVoid != NULL, "sanity");
 
+  if (sig == BREAK_SIGNAL) {
+    assert(!ReduceSignalUsage, "Should not happen with -Xrs/-XX:+ReduceSignalUsage");
+    return true; // ignore it
+  }
+
   // Note: it's not uncommon that JNI code uses signal/sigset to install,
   // then restore certain signal handler (e.g. to temporarily block SIGPIPE,
   // or have a SIGILL handler when detecting CPU type). When that happens,
@@ -1197,7 +1202,7 @@ int os::get_signal_number(const char* signal_name) {
   return -1;
 }
 
-void set_signal_handler(int sig) {
+void set_signal_handler(int sig, bool do_check = true) {
   // Check for overwrite.
   struct sigaction oldAct;
   sigaction(sig, (struct sigaction*)NULL, &oldAct);
@@ -1241,7 +1246,7 @@ void set_signal_handler(int sig) {
 
   // Save handler setup for later checking
   vm_handlers.set(sig, &sigAct);
-  do_check_signal_periodically[sig] = true;
+  do_check_signal_periodically[sig] = do_check;
 
   int ret = sigaction(sig, &sigAct, &oldAct);
   assert(ret == 0, "check");
@@ -1279,7 +1284,12 @@ void install_signal_handlers() {
   set_signal_handler(SIGFPE);
   PPC64_ONLY(set_signal_handler(SIGTRAP);)
   set_signal_handler(SIGXFSZ);
-
+  if (!ReduceSignalUsage) {
+    // This is just for early initialization phase. Intercepting the signal here reduces the risk
+    // that an attach client accidentally forces HotSpot to quit prematurely. We skip the periodic
+    // check because late initialization will overwrite it to UserHandler.
+    set_signal_handler(BREAK_SIGNAL, false);
+  }
 #if defined(__APPLE__)
   // lldb (gdb) installs both standard BSD signal handlers, and mach exception
   // handlers. By replacing the existing task exception handler, we disable lldb's mach