8366298: FDLeakTest sometimes takes minutes to complete on Linux

Reviewed-by: lkorinth, rriggs, stuefe

Author: stefank

test/jdk/java/lang/ProcessBuilder/FDLeakTest/FDLeakTest.java

@@ -27,23 +27,23 @@
  * @summary Check that we don't leak FDs
  * @requires os.family != "windows"
  * @library /test/lib
- * @run main/othervm/native/timeout=480 -Djdk.lang.Process.launchMechanism=posix_spawn -agentlib:FDLeaker FDLeakTest
+ * @run main/othervm/native -Djdk.lang.Process.launchMechanism=posix_spawn -agentlib:FDLeaker FDLeakTest
  */
 
 /**
  * @test id=fork
  * @summary Check that we don't leak FDs
  * @requires os.family != "windows"
  * @library /test/lib
- * @run main/othervm/native/timeout=480 -Djdk.lang.Process.launchMechanism=fork -agentlib:FDLeaker FDLeakTest
+ * @run main/othervm/native -Djdk.lang.Process.launchMechanism=fork -agentlib:FDLeaker FDLeakTest
  */
 
 /**
  * @test id=vfork
  * @summary Check that we don't leak FDs
  * @requires os.family == "linux"
  * @library /test/lib
- * @run main/othervm/native/timeout=480 -Djdk.lang.Process.launchMechanism=vfork -agentlib:FDLeaker FDLeakTest
+ * @run main/othervm/native -Djdk.lang.Process.launchMechanism=vfork -agentlib:FDLeaker FDLeakTest
  */
 
 import jdk.test.lib.process.ProcessTools;

test/jdk/java/lang/ProcessBuilder/FDLeakTest/libFDLeaker.c

@@ -21,16 +21,55 @@
  * questions.
  */
 
+#include <errno.h>
 #include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/resource.h>
 #include "jvmti.h"
 
+static jint limit_num_fds();
+
 JNIEXPORT jint JNICALL
 Agent_OnLoad(JavaVM *jvm, char *options, void *reserved) {
+  // Lower the number of possible open files to make the test go faster
+  jint ret = limit_num_fds();
+  if (ret != 0) {
+    fprintf(stderr, "Failed to limit number of fds: %s", strerror(errno));
+    return ret;
+  }
+
   const char* filename = "./testfile_FDLeaker.txt";
   FILE* f = fopen(filename, "w");
   if (f == NULL) {
+    fprintf(stderr, "Failed to open file: %s", strerror(errno));
     return JNI_ERR;
   }
+
   printf("Opened and leaked %s (%d)", filename, fileno(f));
   return JNI_OK;
 }
+
+static jint limit_num_fds() {
+  struct rlimit rl;
+
+  // Fetch the current limit
+  int ret = getrlimit(RLIMIT_NOFILE, &rl);
+  if (ret != 0) {
+    return JNI_ERR;
+  }
+
+  // Use a lower value unless it is already low
+  rlim_t limit = 100;
+  if (limit < rl.rlim_cur) {
+    rl.rlim_cur = limit;
+  }
+
+  // Lower the value
+  int ret2 = setrlimit(RLIMIT_NOFILE, &rl);
+  if (ret2 != 0) {
+    return JNI_ERR;
+  }
+
+  return JNI_OK;
+}