8324668: JDWP process management needs more efficient file descriptor…

… handling

Reviewed-by: mdoerr
Backport-of: a6632487863db5ff3136cdcc76b7440c15ce6be9

src/jdk.jdwp.agent/unix/native/libjdwp/exec_md.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,12 +23,16 @@
  * questions.
  */
 
+#include <dirent.h>
+#include <errno.h>
 #include <stdlib.h>
+#include <sys/resource.h>
 #include <unistd.h>
 #include <string.h>
 #include <ctype.h>
 #include "sys.h"
 #include "util.h"
+#include "error_messages.h"
 
 static char *skipWhitespace(char *p) {
     while ((*p != '\0') && isspace(*p)) {
@@ -44,6 +48,106 @@ static char *skipNonWhitespace(char *p) {
     return p;
 }
 
+#if defined(_AIX)
+  /* AIX does not understand '/proc/self' - it requires the real process ID */
+  #define FD_DIR aix_fd_dir
+  #define DIR DIR64
+  #define dirent dirent64
+  #define opendir opendir64
+  #define readdir readdir64
+  #define closedir closedir64
+#elif defined(_ALLBSD_SOURCE)
+  #define FD_DIR "/dev/fd"
+#else
+  #define FD_DIR "/proc/self/fd"
+#endif
+
+// Closes every file descriptor that is listed as a directory
+// entry in "/proc/self/fd" (or its equivalent). Standard
+// input/output/error file descriptors will not be closed
+// by this function. This function returns 0 on failure
+// and 1 on success.
+int
+closeDescriptors(void)
+{
+    DIR *dp;
+    struct dirent *dirp;
+    /* leave out standard input/output/error descriptors */
+    int from_fd = STDERR_FILENO + 1;
+
+    /* We're trying to close all file descriptors, but opendir() might
+     * itself be implemented using a file descriptor, and we certainly
+     * don't want to close that while it's in use.  We assume that if
+     * opendir() is implemented using a file descriptor, then it uses
+     * the lowest numbered file descriptor, just like open().  So
+     * before calling opendir(), we close a couple explicitly, so that
+     * opendir() can then use these lowest numbered closed file
+     * descriptors afresh.   */
+
+    close(from_fd);          /* for possible use by opendir() */
+    close(from_fd + 1);      /* another one for good luck */
+    from_fd += 2; /* leave out the 2 we just closed, which the opendir() may use */
+
+#if defined(_AIX)
+    /* set FD_DIR for AIX which does not understand '/proc/self' - it
+     * requires the real process ID */
+    char aix_fd_dir[32];     /* the pid has at most 19 digits */
+    snprintf(aix_fd_dir, 32, "/proc/%d/fd", getpid());
+#endif
+
+    if ((dp = opendir(FD_DIR)) == NULL) {
+        ERROR_MESSAGE(("failed to open dir %s while determining"
+                       " file descriptors to close for process %d",
+                       FD_DIR, getpid()));
+        return 0; // failure
+    }
+
+    while ((dirp = readdir(dp)) != NULL) {
+        if (!isdigit(dirp->d_name[0])) {
+            continue;
+        }
+        const long fd = strtol(dirp->d_name, NULL, 10);
+        if (fd <= INT_MAX && fd >= from_fd) {
+            (void)close((int)fd);
+        }
+    }
+
+    (void)closedir(dp);
+
+    return 1; // success
+}
+
+// Does necessary housekeeping of a forked child process
+// (like closing copied file descriptors) before
+// execing the child process. This function never returns.
+void
+forkedChildProcess(const char *file, char *const argv[])
+{
+    /* Close all file descriptors that have been copied over
+     * from the parent process due to fork(). */
+    if (closeDescriptors() == 0) { /* failed,  close the old way */
+        /* Find max allowed file descriptors for a process
+         * and assume all were opened for the parent process and
+         * copied over to this child process. We close them all. */
+        const rlim_t max_fd = sysconf(_SC_OPEN_MAX);
+        JDI_ASSERT(max_fd != (rlim_t)-1); // -1 represents error
+        /* close(), that we subsequently call, takes only int values */
+        JDI_ASSERT(max_fd <= INT_MAX);
+        /* leave out standard input/output/error file descriptors */
+        rlim_t i = STDERR_FILENO + 1;
+        ERROR_MESSAGE(("failed to close file descriptors of"
+                       " child process optimally, falling back to closing"
+                       " %d file descriptors sequentially", (max_fd - i + 1)));
+        for (; i < max_fd; i++) {
+            (void)close(i);
+        }
+    }
+
+    (void)execvp(file, argv); /* not expected to return */
+
+    exit(errno); /* errno will have been set by the failed execvp */
+}
+
 int
 dbgsysExec(char *cmdLine)
 {
@@ -93,21 +197,11 @@ dbgsysExec(char *cmdLine)
     argv[i] = NULL;  /* NULL terminate */
 
     if ((pid = fork()) == 0) {
-        /* Child process */
-        int i;
-        long max_fd;
-
-        /* close everything */
-        max_fd = sysconf(_SC_OPEN_MAX);
-        /*LINTED*/
-        for (i = 3; i < (int)max_fd; i++) {
-            (void)close(i);
-        }
-
-        (void)execvp(argv[0], argv);
-
-        exit(-1);
+        // manage the child process
+        forkedChildProcess(argv[0], argv);
     }
+    // call to forkedChildProcess(...) will never return for a forked process
+    JDI_ASSERT(pid != 0);
     jvmtiDeallocate(args);
     jvmtiDeallocate(argv);
     if (pid == pid_err) {