8357135: java.lang.OutOfMemoryError: Error creating or attaching to libjvmci after JDK-8356447

Reviewed-by: never, yzheng

Author: Doug Simon

test/hotspot/jtreg/gc/arguments/TestUseCompressedOopsFlagsWithUlimit.java

@@ -54,6 +54,11 @@ private static void checkFlag(long ulimit, long maxram, int maxrampercent, boole
     args.add("-XX:MaxRAM=" + maxram);
     args.add("-XX:MaxRAMPercentage=" + maxrampercent);
     args.add("-XX:+PrintFlagsFinal");
+
+    // Avoid issues with libjvmci failing to reserve
+    // a large virtual address space for its heap
+    args.add("-Xint");
+
     args.add("-version");
 
     // Convert bytes to kbytes for ulimit -v

test/hotspot/jtreg/vmTestbase/nsk/jvmti/Allocate/alloc001/alloc001.java

@@ -85,6 +85,8 @@ public static void main(String[] args) throws Throwable {
                 "-Djava.library.path=" + Utils.TEST_NATIVE_PATH,
                 "-agentpath:" + Utils.TEST_NATIVE_PATH + File.separator + System.mapLibraryName("alloc001"),
                 "-XX:CompressedClassSpaceSize=64m",
+                "-Xint", // Avoids issues with libjvmci failing to reserve
+                         // a large virtual address space for its heap
                 Test.class.getName()
         ));
         cmd = escapeCmd(cmd);