8236569: -Xss not multiple of 4K does not work for the main thread on macOS

Reviewed-by: dholmes, stuefe

Author: asotona

src/java.base/macosx/native/libjli/java_md_macosx.m

@@ -46,6 +46,7 @@
 
 #include <errno.h>
 #include <spawn.h>
+#include <unistd.h>
 
 struct NSAppArgs {
     int argc;
@@ -722,6 +723,20 @@ static void MacOSXStartup(int argc, char *argv[]) {
     return (void*)(intptr_t)JavaMain(args);
 }
 
+static size_t adjustStackSize(size_t stack_size) {
+    long page_size = getpagesize();
+    if (stack_size % page_size == 0) {
+        return stack_size;
+    } else {
+        long pages = stack_size / page_size;
+        // Ensure we don't go over limit
+        if (stack_size <= SIZE_MAX - page_size) {
+            pages++;
+        }
+        return page_size * pages;
+    }
+}
+
 /*
  * Block current thread and continue execution in a new thread.
  */
@@ -734,7 +749,7 @@ static void MacOSXStartup(int argc, char *argv[]) {
     pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
 
     if (stack_size > 0) {
-        pthread_attr_setstacksize(&attr, stack_size);
+        pthread_attr_setstacksize(&attr, adjustStackSize(stack_size));
     }
     pthread_attr_setguardsize(&attr, 0); // no pthread guard page on java threads
 

src/java.base/share/classes/sun/launcher/resources/launcher.properties

@@ -172,6 +172,8 @@ java.launcher.X.usage=\n\
 \                      (Linux Only) show host system or container\n\
 \                      configuration and continue\n\
 \    -Xss<size>        set java thread stack size\n\
+\                      The actual size may be rounded up to a multiple of the\n\
+\                      system page size as required by the operating system.\n\
 \    -Xverify          sets the mode of the bytecode verifier\n\
 \                      Note that option -Xverify:none is deprecated and\n\
 \                      may be removed in a future release.\n\

src/java.base/share/man/java.1

@@ -1151,6 +1151,8 @@ continues.
 Sets the thread stack size (in bytes).
 Append the letter \f[CB]k\f[R] or \f[CB]K\f[R] to indicate KB, \f[CB]m\f[R] or
 \f[CB]M\f[R] to indicate MB, or \f[CB]g\f[R] or \f[CB]G\f[R] to indicate GB.
+The actual size may be rounded up to a multiple of the system page size as
+required by the operating system.
 The default value depends on the platform:
 .RS
 .IP \[bu] 2

src/java.base/unix/native/libjli/java_md.c

@@ -651,6 +651,20 @@ static void* ThreadJavaMain(void* args) {
     return (void*)(intptr_t)JavaMain(args);
 }
 
+static size_t adjustStackSize(size_t stack_size) {
+    long page_size = sysconf(_SC_PAGESIZE);
+    if (stack_size % page_size == 0) {
+        return stack_size;
+    } else {
+        long pages = stack_size / page_size;
+        // Ensure we don't go over limit
+        if (stack_size <= SIZE_MAX - page_size) {
+            pages++;
+        }
+        return page_size * pages;
+    }
+}
+
 /*
  * Block current thread and continue execution in a new thread.
  */
@@ -661,9 +675,17 @@ CallJavaMainInNewThread(jlong stack_size, void* args) {
     pthread_attr_t attr;
     pthread_attr_init(&attr);
     pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
+    size_t adjusted_stack_size;
 
     if (stack_size > 0) {
-        pthread_attr_setstacksize(&attr, stack_size);
+        if (pthread_attr_setstacksize(&attr, stack_size) == EINVAL) {
+            // System may require stack size to be multiple of page size
+            // Retry with adjusted value
+            adjusted_stack_size = adjustStackSize(stack_size);
+            if (adjusted_stack_size != (size_t) stack_size) {
+                pthread_attr_setstacksize(&attr, adjusted_stack_size);
+            }
+        }
     }
     pthread_attr_setguardsize(&attr, 0); // no pthread guard page on java threads